Update extra/product.mk

.drone.yml

@@ -3,7 +3,7 @@ name: default
 
 workspace:
   base: /go
-  path: src/deadbeef.codes/steven/lineageos-ota-server
+  path: src/code.stevenpolley.net/steven/lineageos-ota-server
 
 steps:
 
@@ -22,4 +22,4 @@ steps:
 - name: package in docker container
   image: plugins/docker
   settings:
-    repo: registry.deadbeef.codes/lineageos-ota-server
+    repo: registry.stevenpolley.net/lineageos-ota-server

README.md

@@ -2,31 +2,49 @@
 
 # lineageos-ota-server
 
-OTA Server for LineageOS.  
+A highly-scalable and lightweight OTA Server for LineageOS.  The OTA server supports a single device model (eg: Google Pixel), if you intend to have OTA's available for a variety of device models, just create an additional OTA server instance for each device.  
 
+![alt text][logo]
+
+[logo]: https://deadbeef.codes/steven/lineageos-ota-server/raw/branch/main/ota-logo.jpg "LineageOS OTA Server"
+
+
+### docker compose example
+
+The service listens on port 8080 by default.  Mount the output directory for the builds of the device you wish to serve to the /out folder.  Also, mount a persistent public directory which will be served publicly.  The public folder can be shared across multiple OTA server instances if you wish.  
+
+```yaml
+
+version: '3.8'
+
+  lineageos-ota:
+    image: registry.deadbeef.codes/lineageos-ota-server:latest
+    restart: always
+    expose:
+      - "8080"
+    volumes:
+      - /data/android/lineage/out/target/product/sunfish:/out
+      - /data/android/public/sunfish:/public
+    
+```
 
 ### How to point device to OTA server
 
-We include the URL as a system build.prop by doing the following:
+The recommended way is to include the configuration inside your build of the ROM.  We do this by by including the URL as a system build.prop by doing the following:
 
 Create new file /data/android/lineage/vendor/lineage/build/core/deadbeef-ota.mk
 
-```makefile
-# deadbeef.codes LineageOS OTA update server
-ADDITIONAL_SYSTEM_PROPERTIES += \
-        lineage.updater.uri=https://lineageos-ota.deadbeef.codes
-
-```
 
 
-Edit /data/android/lineage/vendor/lineage/build/core/main.mk to include deadbeef-ota.mk
+Edit (create if not exist) /data/android/lineage/vendor/extra/product.mk
 
 ```makefile
-
-# Include LineageOS versions
-include $(TOPDIR)vendor/lineage/build/core/main_version.mk
-
 # Include deadbeef.codes OTA server
-include $(TOPDIR)vendor/lineage/build/core/deadbeef-ota.mk
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        lineage.updater.uri=https://lineageos-ota-{device}.stevenpolley.net
+
+# Default ADB shell
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        persist.sys.adb.shell=/system_ext/bin/bash
 
 ```

go.mod

@@ -1,3 +1,3 @@
 module deadbeef.codes/steven/lineageos-ota-server
 
-go 1.20
+go 1.22

main.go

@@ -1,15 +1,12 @@
 package main
 
 import (
-	"crypto/sha256"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/fs"
 	"log"
 	"net/http"
 	"os"
-	"strings"
 	"sync"
 )
 
@@ -18,9 +15,9 @@ type LineageOSROM struct {
 	Datetime int    `json:"datetime"` // Unix timestamp - i.e. 1685907926
 	Filename string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
 	ID       string `json:"id"`       // A unique identifier such as a SHA256 hash of the .zip  - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93
-	Romtype  string `json:"romtype"`  // i.e. nightly
+	Romtype  string `json:"romtype"`  // i.e. NIGHTLY or UNOFFICIAL
 	Size     int    `json:"size"`     // size of .zip file in bytes
-	URL      string `json:"url"`      // public URL where client could download the .zip file
+	URL      string `json:"url"`      // Accessible URL where client could download the .zip file
 	Version  string `json:"version"`  // LineageOS version - i.e. 20.0
 }
 
@@ -31,22 +28,53 @@ type HTTPResponseJSON struct {
 
 // Caches data about available ROMs in memory so we don't need to reference the filesystem for each request
 type ROMCache struct {
-	ROMs       []LineageOSROM
+	ROMs         []LineageOSROM  `json:"roms"`
-	Cached     map[string]bool // to quickly lookup if a file is already cached
+	Cached       map[string]bool `json:"-"` // to quickly lookup if a file is already cached
-	sync.Mutex                 // We have multiple goroutines that may be accessing this data simultaneously, so we much lock / unlock it to prevent race conditions
+	sync.RWMutex `json:"-"`      // We have multiple goroutines that may be accessing this data simultaneously, so we much lock / unlock it to prevent race conditions
 }
 
-var ( // evil global variable
+const (
+	romDirectory      = "public"               // directory where ROMs are available for download
+	buildOutDirectory = "out"                  // directory from build system containing artifacts which we can move to romDirectory
+	cacheFile         = "public/romcache.json" // persistence between server restarts so we don't have to rehash all the ROM files each time the program starts
+)
+
+var ( // evil global variables
 	romCache ROMCache
+	baseURL  string
 )
 
 func init() {
+	// intialize and load ROMCache from file - so we don't have to rehash all the big files again
 	romCache = ROMCache{}
+	baseURL = os.Getenv("baseurl")
+	if len(baseURL) < 1 {
+		log.Fatalf("required environment variable 'baseurl' is not set.")
+	}
+
+	romCacheJson, err := os.ReadFile(cacheFile)
+	if err != nil {
+		if err != os.ErrNotExist { // don't care if it doesn't exist, just skip it
+			log.Printf("failed to read romCache file : %v", err)
+		}
+	} else { // if opening the file's successful, then load the contents
+		err = json.Unmarshal(romCacheJson, &romCache)
+		if err != nil {
+			log.Printf("failed to unmarshal romCacheJson to romCache struct: %v", err)
+		}
+
+	}
+
 	romCache.Cached = make(map[string]bool)
 
-	// Check if any new build artifacts and preload the romCache
+	for _, rom := range romCache.ROMs {
-	moveBuildArtifacts("out", "public")
+		romCache.Cached[rom.Filename] = true
-	go updateROMCache("public")
+		log.Printf("loaded cached file: %s", rom.Filename)
+	}
+
+	// Check if any new build artifacts and load any new files into the romCache
+	moveBuildArtifacts()
+	go updateROMCache()
 }
 
 // HTTP Server
@@ -63,7 +91,7 @@ func main() {
 }
 
 // Reads the ROM files on the filesystem and populates a slice of linageOSROMs
-func updateROMCache(romDirectory string) {
+func updateROMCache() {
 
 	log.Printf("updating ROM cache")
 
@@ -79,22 +107,26 @@ func updateROMCache(romDirectory string) {
 		return
 	}
 
+	wg := sync.WaitGroup{}
+
 	for i, v := range files {
 
-		isLineageROM, splitName := isLineageROMZip(v)
+		isLineageROM, splitName := parseROMFileName(v)
 		if !isLineageROM {
 			continue
 		}
 
 		// skip already cached files
-		romCache.Lock()
+		romCache.RLock()
 		if _, ok := romCache.Cached[v.Name()]; ok {
-			romCache.Unlock()
+			romCache.RUnlock()
 			continue
 		}
-		romCache.Unlock()
+		romCache.RUnlock()
 
+		wg.Add(1)
 		go func(v fs.DirEntry) {
+			defer wg.Done()
 			fInfo, err := v.Info()
 			if err != nil {
 				log.Printf("failed to get file info '%s': %v", v.Name(), err)
@@ -111,9 +143,9 @@ func updateROMCache(romDirectory string) {
 				Datetime: int(fInfo.ModTime().Unix()),
 				Filename: v.Name(),
 				ID:       fileHash,
-				Romtype:  "nightly",
+				Romtype:  splitName[3], // UNOFFICIAL
 				Size:     int(fInfo.Size()),
-				URL:      fmt.Sprintf("https://lineageos-ota.deadbeef.codes/public/%s", v.Name()),
+				URL:      fmt.Sprintf("%s/public/%s", baseURL, v.Name()),
 				Version:  splitName[1],
 			}
 
@@ -125,75 +157,47 @@ func updateROMCache(romDirectory string) {
 			romCache.ROMs = append(romCache.ROMs, lineageOSROM)
 			romCache.Cached[v.Name()] = true
 			romCache.Unlock()
+
 		}(files[i])
 	}
-}
 
-// returns true if new builds were moved
+	// save file to disk for next startup so we don't have to rehash all the files again
-func moveBuildArtifacts(outDirectory, romDirectory string) bool {
+	wg.Wait()
-
+	romCache.RLock()
-	f, err := os.Open(outDirectory)
+	romCacheJson, err := json.Marshal(romCache)
+	romCache.RUnlock()
 	if err != nil {
-		log.Printf("failed to open ROM directory: %v", err)
+		log.Printf("failed to marshal romCache to json: %v", err)
-		return false
+		return
 	}
-	defer f.Close()
+
-	files, err := f.ReadDir(0)
+	err = os.WriteFile(cacheFile, romCacheJson, 0644)
 	if err != nil {
-		log.Printf("failed to read files in directory: %v", err)
+		log.Printf("failed to write '%s' file: %v", cacheFile, err)
-		return false
+		log.Printf("attempting to remove '%s' to ensure integrity during next program startup...", cacheFile)
-	}
+		err = os.Remove(cacheFile)
-
-	var newROMs bool
-
-	for _, v := range files {
-		if isLineageROM, _ := isLineageROMZip(v); !isLineageROM { // skip files that aren't LineageOS ROMs
-			continue
-		}
-
-		newROMs = true
-		log.Printf("new build found - moving file %s", v.Name())
-		romCache.Lock() // lock to prevent multiple concurrent goroutines moving the same file
-		err := moveBuildFile(fmt.Sprintf("%s/%s", outDirectory, v.Name()), fmt.Sprintf("%s/%s", romDirectory, v.Name()))
-		romCache.Unlock()
 		if err != nil {
-			log.Printf("failed to move file '%s' from out to rom directory: %v", v.Name(), err)
+			log.Printf("failed to remove file '%s': %v", cacheFile, err)
-			continue
 		}
+		return
 	}
-
-	return newROMs
-}
-
-// false if a file is not a LineageOS ROM .zip file
-// no formal validation is performed - only file naming convention is checked
-// also returns a lineage ROM's filename sliced and delimited by -'s
-// Example filename: lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
-func isLineageROMZip(v fs.DirEntry) (bool, []string) {
-
-	// skip directories, non .zip files and files that don't begin with lineage-
-	if v.Type().IsDir() || !strings.HasSuffix(v.Name(), ".zip") || !strings.HasPrefix(v.Name(), "lineage-") {
-		return false, nil
-	}
-
-	splitName := strings.Split(v.Name(), "-")
-	// expect 5 dashes
-	return len(splitName) == 5, splitName
 }
 
 // http - GET /
-// Writes JSON response for the updater app to know what versions are available to download
+// The LineageOS updater app needs a JSON array of type LineageOSROM
+// Marshal's romCache.ROMs to JSON to serve as the response body
+
 func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
-	go func() {
+	go func() { // Also checks for new builds - TBD need a better method as the first request will return no new updates.  inotify?
-		newBuilds := moveBuildArtifacts("out", "public")
+		newBuilds := moveBuildArtifacts()
 		if newBuilds {
-			updateROMCache("public")
+			updateROMCache()
 		}
 	}()
 
-	romCache.Lock()
+	romCache.RLock()
 	httpResponseJSON := &HTTPResponseJSON{Response: romCache.ROMs}
-	romCache.Unlock()
+	romCache.RUnlock()
 
 	b, err := json.Marshal(httpResponseJSON)
 	if err != nil {
@@ -204,55 +208,3 @@ func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
 
 	w.Write(b)
 }
-
-// Returns a sha256 hash of a file located at the path provided
-func hashFile(filename string) (string, error) {
-	f, err := os.Open(filename)
-	if err != nil {
-		return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err)
-	}
-	defer f.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}
-
-// A custom "move file" function because in docker container the mounted folders are different overlay filesystems
-// Instead of os.Rename, we must copy and delete
-func moveBuildFile(src, dst string) error {
-	sourceFileStat, err := os.Stat(src)
-	if err != nil {
-		return err
-	}
-
-	if !sourceFileStat.Mode().IsRegular() {
-		return fmt.Errorf("%s is not a regular file", src)
-	}
-
-	source, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer source.Close()
-
-	destination, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer destination.Close()
-	_, err = io.Copy(destination, source)
-	if err != nil {
-		return fmt.Errorf("failed to copy file: %v", err)
-	}
-
-	err = os.Remove(src)
-	if err != nil {
-		return fmt.Errorf("failed to delete source file after copy: %v", err)
-	}
-
-	return nil
-}

processFiles.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"strings"
+)
+
+// Searches the build toolchain output directory for new LineageOS builds.
+// Any new builds are moved into the public http server directory.
+// Returns true if new builds were moved
+func moveBuildArtifacts() bool {
+
+	f, err := os.Open(buildOutDirectory)
+	if err != nil {
+		log.Printf("failed to open ROM directory: %v", err)
+		return false
+	}
+	defer f.Close()
+	files, err := f.ReadDir(0)
+	if err != nil {
+		log.Printf("failed to read files in directory: %v", err)
+		return false
+	}
+
+	var newROMs bool
+
+	for _, v := range files {
+		if isLineageROM, _ := parseROMFileName(v); !isLineageROM { // skip files that aren't LineageOS ROMs
+			continue
+		}
+
+		newROMs = true
+		log.Printf("new build found - moving file %s", v.Name())
+		romCache.Lock() // RW lock to prevent multiple concurrent goroutines moving the same file
+		err := copyThenDeleteFile(fmt.Sprintf("%s/%s", buildOutDirectory, v.Name()), fmt.Sprintf("%s/%s", romDirectory, v.Name()))
+		romCache.Unlock()
+		if err != nil {
+			log.Printf("failed to move file '%s' from out to rom directory: %v", v.Name(), err)
+			continue
+		}
+	}
+
+	return newROMs
+}
+
+// Returns a sha256 hash of a file located at the path provided
+func hashFile(filename string) (string, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err)
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+// false if a file is not a LineageOS ROM .zip file
+// no formal validation is performed - only file naming convention is checked
+// also returns a lineage ROM's filename sliced and delimited by -'s
+// Example filename: lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
+func parseROMFileName(v fs.DirEntry) (bool, []string) {
+
+	// skip directories, non .zip files and files that don't begin with lineage-
+	if v.Type().IsDir() || !strings.HasSuffix(v.Name(), ".zip") || !strings.HasPrefix(v.Name(), "lineage-") {
+		return false, nil
+	}
+
+	splitName := strings.Split(v.Name(), "-")
+	// expect 5 dashes
+	return len(splitName) == 5, splitName
+}
+
+// A custom "move file" function because in docker container the mounted folders are different overlay filesystems
+// Instead of os.Rename, we must copy and delete
+func copyThenDeleteFile(src, dst string) error {
+	sourceFileStat, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", src)
+	}
+
+	source, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+	_, err = io.Copy(destination, source)
+	if err != nil {
+		return fmt.Errorf("failed to copy file: %v", err)
+	}
+
+	err = os.Remove(src)
+	if err != nil {
+		return fmt.Errorf("failed to delete source file after copy: %v", err)
+	}
+
+	return nil
+}