Update extra/product.mk

.drone.yml

@@ -3,7 +3,7 @@ name: default
 
 workspace:
   base: /go
-  path: src/deadbeef.codes/steven/lineageos-ota-server
+  path: src/code.stevenpolley.net/steven/lineageos-ota-server
 
 steps:
 
@@ -22,4 +22,4 @@ steps:
 - name: package in docker container
   image: plugins/docker
   settings:
-    repo: registry.deadbeef.codes/lineageos-ota-server
+    repo: registry.stevenpolley.net/lineageos-ota-server

README.md

@@ -2,31 +2,49 @@
 
 # lineageos-ota-server
 
-OTA Server for LineageOS.  
+A highly-scalable and lightweight OTA Server for LineageOS.  The OTA server supports a single device model (eg: Google Pixel), if you intend to have OTA's available for a variety of device models, just create an additional OTA server instance for each device.  
 
+![alt text][logo]
+
+[logo]: https://deadbeef.codes/steven/lineageos-ota-server/raw/branch/main/ota-logo.jpg "LineageOS OTA Server"
+
+
+### docker compose example
+
+The service listens on port 8080 by default.  Mount the output directory for the builds of the device you wish to serve to the /out folder.  Also, mount a persistent public directory which will be served publicly.  The public folder can be shared across multiple OTA server instances if you wish.  
+
+```yaml
+
+version: '3.8'
+
+  lineageos-ota:
+    image: registry.deadbeef.codes/lineageos-ota-server:latest
+    restart: always
+    expose:
+      - "8080"
+    volumes:
+      - /data/android/lineage/out/target/product/sunfish:/out
+      - /data/android/public/sunfish:/public
+    
+```
 
 ### How to point device to OTA server
 
-We include the URL as a system build.prop by doing the following:
+The recommended way is to include the configuration inside your build of the ROM.  We do this by by including the URL as a system build.prop by doing the following:
 
 Create new file /data/android/lineage/vendor/lineage/build/core/deadbeef-ota.mk
 
-```makefile
-# deadbeef.codes LineageOS OTA update server
-ADDITIONAL_SYSTEM_PROPERTIES += \
-        lineage.updater.uri=https://lineageos-ota.deadbeef.codes
-
-```
 
 
-Edit /data/android/lineage/vendor/lineage/build/core/main.mk to include deadbeef-ota.mk
+Edit (create if not exist) /data/android/lineage/vendor/extra/product.mk
 
 ```makefile
-
-# Include LineageOS versions
-include $(TOPDIR)vendor/lineage/build/core/main_version.mk
-
 # Include deadbeef.codes OTA server
-include $(TOPDIR)vendor/lineage/build/core/deadbeef-ota.mk
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        lineage.updater.uri=https://lineageos-ota-{device}.stevenpolley.net
+
+# Default ADB shell
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        persist.sys.adb.shell=/system_ext/bin/bash
 
 ```

go.mod

@@ -1,3 +1,3 @@
 module deadbeef.codes/steven/lineageos-ota-server
 
-go 1.20
+go 1.22

lineage-android-build-files/README.md

@@ -1,7 +0,0 @@
-These files need to be deployed into your Android source code directory to ensure build.prop file on the device contains the correct URL for receiving OTA updates.  This is to point the updated app at the OTA server you're hosting.
-
-1. Create the directory lineage/vendor/extra
-2. Review and edit the contents of file product.prop to ensure the URL is what you want your device pointed to.
-3. Place the product.mk and product.prop files in this directory
-
-The files will then be automatically read by the build system and patch build.prop with the updated URL.

lineage-android-build-files/product.mk

@@ -1 +0,0 @@
-TARGET_PRODUCT_PROP += $(COMMON_PATH)/product.prop

lineage-android-build-files/product.prop

@@ -1 +0,0 @@
-lineage.updater.uri=https://lineage-ota.deadbeef.codes

main.go

@@ -1,27 +1,24 @@
 package main
 
 import (
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/fs"
 	"log"
 	"net/http"
 	"os"
-	"strings"
 	"sync"
 )
 
 // Information for a LineageOS ROM available for download
 type LineageOSROM struct {
-	Datetime int    `json:"datetime"`
+	Datetime int    `json:"datetime"` // Unix timestamp - i.e. 1685907926
-	Filename string `json:"filename"`
+	Filename string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
-	ID       string `json:"id"`
+	ID       string `json:"id"`       // A unique identifier such as a SHA256 hash of the .zip  - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93
-	Romtype  string `json:"romtype"`
+	Romtype  string `json:"romtype"`  // i.e. NIGHTLY or UNOFFICIAL
-	Size     int    `json:"size"`
+	Size     int    `json:"size"`     // size of .zip file in bytes
-	URL      string `json:"url"`
+	URL      string `json:"url"`      // Accessible URL where client could download the .zip file
-	Version  string `json:"version"`
+	Version  string `json:"version"`  // LineageOS version - i.e. 20.0
 }
 
 // The HTTP response JSON should be a JSON array of lineageOSROMS available for download
@@ -29,26 +26,58 @@ type HTTPResponseJSON struct {
 	Response []LineageOSROM `json:"response"`
 }
 
+// Caches data about available ROMs in memory so we don't need to reference the filesystem for each request
 type ROMCache struct {
-	ROMs   []LineageOSROM
+	ROMs         []LineageOSROM  `json:"roms"`
-	Cached map[string]bool // to quickly lookup if a file is already cached
+	Cached       map[string]bool `json:"-"` // to quickly lookup if a file is already cached
-	sync.Mutex
+	sync.RWMutex `json:"-"`      // We have multiple goroutines that may be accessing this data simultaneously, so we much lock / unlock it to prevent race conditions
 }
 
-var (
+const (
-	romCache ROMCache
+	romDirectory      = "public"               // directory where ROMs are available for download
+	buildOutDirectory = "out"                  // directory from build system containing artifacts which we can move to romDirectory
+	cacheFile         = "public/romcache.json" // persistence between server restarts so we don't have to rehash all the ROM files each time the program starts
+)
+
+var ( // evil global variables
+	romCache ROMCache
+	baseURL  string
 )
 
-// Preload cached list of files and hashes
 func init() {
+	// intialize and load ROMCache from file - so we don't have to rehash all the big files again
 	romCache = ROMCache{}
+	baseURL = os.Getenv("baseurl")
+	if len(baseURL) < 1 {
+		log.Fatalf("required environment variable 'baseurl' is not set.")
+	}
+
+	romCacheJson, err := os.ReadFile(cacheFile)
+	if err != nil {
+		if err != os.ErrNotExist { // don't care if it doesn't exist, just skip it
+			log.Printf("failed to read romCache file : %v", err)
+		}
+	} else { // if opening the file's successful, then load the contents
+		err = json.Unmarshal(romCacheJson, &romCache)
+		if err != nil {
+			log.Printf("failed to unmarshal romCacheJson to romCache struct: %v", err)
+		}
+
+	}
+
 	romCache.Cached = make(map[string]bool)
 
-	moveBuildArtifacts("out", "public")
+	for _, rom := range romCache.ROMs {
-	go updateROMCache("public")
+		romCache.Cached[rom.Filename] = true
+		log.Printf("loaded cached file: %s", rom.Filename)
+	}
+
+	// Check if any new build artifacts and load any new files into the romCache
+	moveBuildArtifacts()
+	go updateROMCache()
 }
 
-// HTTP Routing
+// HTTP Server
 func main() {
 
 	//Public static files
@@ -59,11 +88,10 @@ func main() {
 
 	log.Print("Service listening on :8080")
 	log.Fatal(http.ListenAndServe(":8080", nil))
-
 }
 
 // Reads the ROM files on the filesystem and populates a slice of linageOSROMs
-func updateROMCache(romDirectory string) {
+func updateROMCache() {
 
 	log.Printf("updating ROM cache")
 
@@ -79,32 +107,26 @@ func updateROMCache(romDirectory string) {
 		return
 	}
 
+	wg := sync.WaitGroup{}
+
 	for i, v := range files {
 
-		// skip directories
+		isLineageROM, splitName := parseROMFileName(v)
-		if v.Type().IsDir() {
+		if !isLineageROM {
-			continue
-		}
-
-		// skip non .zip files
-		if !strings.HasSuffix(v.Name(), ".zip") {
 			continue
 		}
 
 		// skip already cached files
-		romCache.Lock()
+		romCache.RLock()
 		if _, ok := romCache.Cached[v.Name()]; ok {
-			romCache.Unlock()
+			romCache.RUnlock()
 			continue
 		}
-		romCache.Unlock()
+		romCache.RUnlock()
 
-		// Parse filename and skip files that can't be parsed
+		wg.Add(1)
-		splitName := strings.Split(v.Name(), "-")
-		if len(splitName) != 5 {
-			continue
-		}
 		go func(v fs.DirEntry) {
+			defer wg.Done()
 			fInfo, err := v.Info()
 			if err != nil {
 				log.Printf("failed to get file info '%s': %v", v.Name(), err)
@@ -121,9 +143,9 @@ func updateROMCache(romDirectory string) {
 				Datetime: int(fInfo.ModTime().Unix()),
 				Filename: v.Name(),
 				ID:       fileHash,
-				Romtype:  "nightly",
+				Romtype:  splitName[3], // UNOFFICIAL
 				Size:     int(fInfo.Size()),
-				URL:      fmt.Sprintf("https://lineageos-ota.deadbeef.codes/public/%s", v.Name()),
+				URL:      fmt.Sprintf("%s/public/%s", baseURL, v.Name()),
 				Version:  splitName[1],
 			}
 
@@ -135,61 +157,47 @@ func updateROMCache(romDirectory string) {
 			romCache.ROMs = append(romCache.ROMs, lineageOSROM)
 			romCache.Cached[v.Name()] = true
 			romCache.Unlock()
+
 		}(files[i])
 	}
-}
 
-// returns true if new builds were moved
+	// save file to disk for next startup so we don't have to rehash all the files again
-func moveBuildArtifacts(outDirectory, romDirectory string) bool {
+	wg.Wait()
-
+	romCache.RLock()
-	f, err := os.Open(outDirectory)
+	romCacheJson, err := json.Marshal(romCache)
+	romCache.RUnlock()
 	if err != nil {
-		log.Printf("failed to open ROM directory: %v", err)
+		log.Printf("failed to marshal romCache to json: %v", err)
-		return false
+		return
-	}
-	defer f.Close()
-	files, err := f.ReadDir(0)
-	if err != nil {
-		log.Printf("failed to read files in directory: %v", err)
-		return false
 	}
 
-	var newROMs bool
+	err = os.WriteFile(cacheFile, romCacheJson, 0644)
-
+	if err != nil {
-	for _, v := range files {
+		log.Printf("failed to write '%s' file: %v", cacheFile, err)
-		// skip directories
+		log.Printf("attempting to remove '%s' to ensure integrity during next program startup...", cacheFile)
-		if v.Type().IsDir() {
+		err = os.Remove(cacheFile)
-			continue
-		}
-		// skip non .zip files
-		if !strings.HasSuffix(v.Name(), ".zip") {
-			continue
-		}
-		// Parse filename and skip files that can't be parsed
-		splitName := strings.Split(v.Name(), "-")
-		if len(splitName) != 5 {
-			continue
-		}
-		newROMs = true
-		log.Printf("new build found - moving file %s", v.Name())
-		err := moveBuildFile(fmt.Sprintf("%s/%s", outDirectory, v.Name()), fmt.Sprintf("%s/%s", romDirectory, v.Name()))
 		if err != nil {
-			log.Printf("failed to move file '%s' from out to rom directory: %v", v.Name(), err)
+			log.Printf("failed to remove file '%s': %v", cacheFile, err)
-			continue
 		}
+		return
 	}
-
-	return newROMs
 }
 
 // http - GET /
-// Writes JSON response for the updater app to know what versions are available to download
+// The LineageOS updater app needs a JSON array of type LineageOSROM
-func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
+// Marshal's romCache.ROMs to JSON to serve as the response body
-	romCache.Lock()
-	lineageOSROMs := romCache.ROMs
-	romCache.Unlock()
 
-	httpResponseJSON := &HTTPResponseJSON{Response: lineageOSROMs}
+func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
+	go func() { // Also checks for new builds - TBD need a better method as the first request will return no new updates.  inotify?
+		newBuilds := moveBuildArtifacts()
+		if newBuilds {
+			updateROMCache()
+		}
+	}()
+
+	romCache.RLock()
+	httpResponseJSON := &HTTPResponseJSON{Response: romCache.ROMs}
+	romCache.RUnlock()
 
 	b, err := json.Marshal(httpResponseJSON)
 	if err != nil {
@@ -199,62 +207,4 @@ func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	w.Write(b)
-
-	go func() {
-		newBuilds := moveBuildArtifacts("out", "public")
-		if newBuilds {
-			updateROMCache("public")
-		}
-	}()
-}
-
-func hashFile(filename string) (string, error) {
-	f, err := os.Open(filename)
-	if err != nil {
-		return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err)
-	}
-	defer f.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}
-
-// A custom "move file" function because in docker container the mounted folders are different overlay filesystems
-// Instead of os.Rename, we must copy and delete
-func moveBuildFile(src, dst string) error {
-	sourceFileStat, err := os.Stat(src)
-	if err != nil {
-		return err
-	}
-
-	if !sourceFileStat.Mode().IsRegular() {
-		return fmt.Errorf("%s is not a regular file", src)
-	}
-
-	source, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer source.Close()
-
-	destination, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer destination.Close()
-	_, err = io.Copy(destination, source)
-	if err != nil {
-		return fmt.Errorf("failed to copy file: %v", err)
-	}
-
-	err = os.Remove(src)
-	if err != nil {
-		return fmt.Errorf("failed to delete source file after copy: %v", err)
-	}
-
-	return nil
 }

processFiles.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"strings"
+)
+
+// Searches the build toolchain output directory for new LineageOS builds.
+// Any new builds are moved into the public http server directory.
+// Returns true if new builds were moved
+func moveBuildArtifacts() bool {
+
+	f, err := os.Open(buildOutDirectory)
+	if err != nil {
+		log.Printf("failed to open ROM directory: %v", err)
+		return false
+	}
+	defer f.Close()
+	files, err := f.ReadDir(0)
+	if err != nil {
+		log.Printf("failed to read files in directory: %v", err)
+		return false
+	}
+
+	var newROMs bool
+
+	for _, v := range files {
+		if isLineageROM, _ := parseROMFileName(v); !isLineageROM { // skip files that aren't LineageOS ROMs
+			continue
+		}
+
+		newROMs = true
+		log.Printf("new build found - moving file %s", v.Name())
+		romCache.Lock() // RW lock to prevent multiple concurrent goroutines moving the same file
+		err := copyThenDeleteFile(fmt.Sprintf("%s/%s", buildOutDirectory, v.Name()), fmt.Sprintf("%s/%s", romDirectory, v.Name()))
+		romCache.Unlock()
+		if err != nil {
+			log.Printf("failed to move file '%s' from out to rom directory: %v", v.Name(), err)
+			continue
+		}
+	}
+
+	return newROMs
+}
+
+// Returns a sha256 hash of a file located at the path provided
+func hashFile(filename string) (string, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err)
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+// false if a file is not a LineageOS ROM .zip file
+// no formal validation is performed - only file naming convention is checked
+// also returns a lineage ROM's filename sliced and delimited by -'s
+// Example filename: lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
+func parseROMFileName(v fs.DirEntry) (bool, []string) {
+
+	// skip directories, non .zip files and files that don't begin with lineage-
+	if v.Type().IsDir() || !strings.HasSuffix(v.Name(), ".zip") || !strings.HasPrefix(v.Name(), "lineage-") {
+		return false, nil
+	}
+
+	splitName := strings.Split(v.Name(), "-")
+	// expect 5 dashes
+	return len(splitName) == 5, splitName
+}
+
+// A custom "move file" function because in docker container the mounted folders are different overlay filesystems
+// Instead of os.Rename, we must copy and delete
+func copyThenDeleteFile(src, dst string) error {
+	sourceFileStat, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", src)
+	}
+
+	source, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+	_, err = io.Copy(destination, source)
+	if err != nil {
+		return fmt.Errorf("failed to copy file: %v", err)
+	}
+
+	err = os.Remove(src)
+	if err != nil {
+		return fmt.Errorf("failed to delete source file after copy: %v", err)
+	}
+
+	return nil
+}