Update extra/product.mk

.drone.yml

@@ -3,7 +3,7 @@ name: default
 
 workspace:
   base: /go
-  path: src/deadbeef.codes/steven/lineageos-ota-server
+  path: src/code.stevenpolley.net/steven/lineageos-ota-server
 
 steps:
 
@@ -22,4 +22,4 @@ steps:
 - name: package in docker container
   image: plugins/docker
   settings:
-    repo: registry.deadbeef.codes/lineageos-ota-server
+    repo: registry.stevenpolley.net/lineageos-ota-server

README.md

@@ -2,4 +2,49 @@
 
 # lineageos-ota-server
 
-OTA Server for LineageOS
+A highly-scalable and lightweight OTA Server for LineageOS.  The OTA server supports a single device model (eg: Google Pixel), if you intend to have OTA's available for a variety of device models, just create an additional OTA server instance for each device.  
+
+![alt text][logo]
+
+[logo]: https://deadbeef.codes/steven/lineageos-ota-server/raw/branch/main/ota-logo.jpg "LineageOS OTA Server"
+
+
+### docker compose example
+
+The service listens on port 8080 by default.  Mount the output directory for the builds of the device you wish to serve to the /out folder.  Also, mount a persistent public directory which will be served publicly.  The public folder can be shared across multiple OTA server instances if you wish.  
+
+```yaml
+
+version: '3.8'
+
+  lineageos-ota:
+    image: registry.deadbeef.codes/lineageos-ota-server:latest
+    restart: always
+    expose:
+      - "8080"
+    volumes:
+      - /data/android/lineage/out/target/product/sunfish:/out
+      - /data/android/public/sunfish:/public
+    
+```
+
+### How to point device to OTA server
+
+The recommended way is to include the configuration inside your build of the ROM.  We do this by by including the URL as a system build.prop by doing the following:
+
+Create new file /data/android/lineage/vendor/lineage/build/core/deadbeef-ota.mk
+
+
+
+Edit (create if not exist) /data/android/lineage/vendor/extra/product.mk
+
+```makefile
+# Include deadbeef.codes OTA server
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        lineage.updater.uri=https://lineageos-ota-{device}.stevenpolley.net
+
+# Default ADB shell
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        persist.sys.adb.shell=/system_ext/bin/bash
+
+```

go.mod

@@ -1,3 +1,3 @@
 module deadbeef.codes/steven/lineageos-ota-server
 
-go 1.20
+go 1.22

main.go

@@ -7,27 +7,77 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
-	"strings"
+	"sync"
 )
 
 // Information for a LineageOS ROM available for download
 type LineageOSROM struct {
-	Datetime int    `json:"datetime"`
-	Filename string `json:"filename"`
-	ID       string `json:"id"`
-	Romtype  string `json:"romtype"`
-	Size     int    `json:"size"`
-	URL      string `json:"url"`
-	Version  string `json:"version"`
+	Datetime int    `json:"datetime"` // Unix timestamp - i.e. 1685907926
+	Filename string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
+	ID       string `json:"id"`       // A unique identifier such as a SHA256 hash of the .zip  - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93
+	Romtype  string `json:"romtype"`  // i.e. NIGHTLY or UNOFFICIAL
+	Size     int    `json:"size"`     // size of .zip file in bytes
+	URL      string `json:"url"`      // Accessible URL where client could download the .zip file
+	Version  string `json:"version"`  // LineageOS version - i.e. 20.0
 }
 
 // The HTTP response JSON should be a JSON array of lineageOSROMS available for download
-type HttpResponseJSON struct {
+type HTTPResponseJSON struct {
 	Response []LineageOSROM `json:"response"`
 }
 
-// HTTP Routing
+// Caches data about available ROMs in memory so we don't need to reference the filesystem for each request
+type ROMCache struct {
+	ROMs         []LineageOSROM  `json:"roms"`
+	Cached       map[string]bool `json:"-"` // to quickly lookup if a file is already cached
+	sync.RWMutex `json:"-"`      // We have multiple goroutines that may be accessing this data simultaneously, so we much lock / unlock it to prevent race conditions
+}
+
+const (
+	romDirectory      = "public"               // directory where ROMs are available for download
+	buildOutDirectory = "out"                  // directory from build system containing artifacts which we can move to romDirectory
+	cacheFile         = "public/romcache.json" // persistence between server restarts so we don't have to rehash all the ROM files each time the program starts
+)
+
+var ( // evil global variables
+	romCache ROMCache
+	baseURL  string
+)
+
+func init() {
+	// intialize and load ROMCache from file - so we don't have to rehash all the big files again
+	romCache = ROMCache{}
+	baseURL = os.Getenv("baseurl")
+	if len(baseURL) < 1 {
+		log.Fatalf("required environment variable 'baseurl' is not set.")
+	}
+
+	romCacheJson, err := os.ReadFile(cacheFile)
+	if err != nil {
+		if err != os.ErrNotExist { // don't care if it doesn't exist, just skip it
+			log.Printf("failed to read romCache file : %v", err)
+		}
+	} else { // if opening the file's successful, then load the contents
+		err = json.Unmarshal(romCacheJson, &romCache)
+		if err != nil {
+			log.Printf("failed to unmarshal romCacheJson to romCache struct: %v", err)
+		}
+
+	}
+
+	romCache.Cached = make(map[string]bool)
+
+	for _, rom := range romCache.ROMs {
+		romCache.Cached[rom.Filename] = true
+		log.Printf("loaded cached file: %s", rom.Filename)
+	}
+
+	// Check if any new build artifacts and load any new files into the romCache
+	moveBuildArtifacts()
+	go updateROMCache()
+}
+
+// HTTP Server
 func main() {
 
 	//Public static files
@@ -38,71 +88,116 @@ func main() {
 
 	log.Print("Service listening on :8080")
 	log.Fatal(http.ListenAndServe(":8080", nil))
-
 }
 
 // Reads the ROM files on the filesystem and populates a slice of linageOSROMs
-func getLineageOSROMs(romDirectory string) ([]LineageOSROM, error) {
+func updateROMCache() {
 
-	if _, err := os.Stat(romDirectory); os.IsNotExist(err) {
-		return nil, fmt.Errorf("romDirectory '%s' does not exist", romDirectory)
-	}
+	log.Printf("updating ROM cache")
 
-	// Get all the zip files and populate romFileNames slice
-	var lineageOSROMs []LineageOSROM
-
-	err := filepath.WalkDir(romDirectory, func(s string, d fs.DirEntry, err error) error {
+	f, err := os.Open(romDirectory)
 	if err != nil {
-			return fmt.Errorf("walk error occured during file '%s': %v", d.Name(), err)
+		log.Printf("failed to open ROM directory: %v", err)
+		return
 	}
-		if filepath.Ext(d.Name()) != ".zip" {
-			return nil
-		}
-
-		fInfo, err := d.Info()
+	defer f.Close()
+	files, err := f.ReadDir(0)
 	if err != nil {
-			return fmt.Errorf("failed to get file info '%s': %v", d.Name(), err)
+		log.Printf("failed to read files in directory: %v", err)
+		return
 	}
 
-		// Get information about file and populate rom
+	wg := sync.WaitGroup{}
 
-		splitName := strings.Split(d.Name(), "-")
-		if len(splitName) != 5 {
-			log.Printf("ignoring zip file '%d', name is not formatted correctly ")
+	for i, v := range files {
+
+		isLineageROM, splitName := parseROMFileName(v)
+		if !isLineageROM {
+			continue
+		}
+
+		// skip already cached files
+		romCache.RLock()
+		if _, ok := romCache.Cached[v.Name()]; ok {
+			romCache.RUnlock()
+			continue
+		}
+		romCache.RUnlock()
+
+		wg.Add(1)
+		go func(v fs.DirEntry) {
+			defer wg.Done()
+			fInfo, err := v.Info()
+			if err != nil {
+				log.Printf("failed to get file info '%s': %v", v.Name(), err)
+				return
+			}
+
+			fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, v.Name()))
+			if err != nil {
+				log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", v.Name(), err)
+				return
 			}
 
 			lineageOSROM := LineageOSROM{
 				Datetime: int(fInfo.ModTime().Unix()),
-			Filename: d.Name(),
-			ID:       "TBD",
-			Romtype:  "nightly",
+				Filename: v.Name(),
+				ID:       fileHash,
+				Romtype:  splitName[3], // UNOFFICIAL
 				Size:     int(fInfo.Size()),
-			URL:      fmt.Sprintf("https://lineageos-updater.deadbeef.codes/public/%s", d.Name()),
-			Version:  "TBD",
+				URL:      fmt.Sprintf("%s/public/%s", baseURL, v.Name()),
+				Version:  splitName[1],
 			}
 
-		lineageOSROMs = append(lineageOSROMs, lineageOSROM)
+			romCache.Lock()
+			if _, ok := romCache.Cached[v.Name()]; ok { // it's possible another goroutine was already working to cache the file, so we check at the end
+				romCache.Unlock()
+				return
+			}
+			romCache.ROMs = append(romCache.ROMs, lineageOSROM)
+			romCache.Cached[v.Name()] = true
+			romCache.Unlock()
 
-		return nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to walk romDirectory '%s': %v", romDirectory, err)
+		}(files[i])
 	}
 
-	return lineageOSROMs, nil
-}
-
-// http - GET /
-// Writes JSON response for the updater app to know what versions are available to download
-func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
-	lineageOSROMs, err := getLineageOSROMs("public")
+	// save file to disk for next startup so we don't have to rehash all the files again
+	wg.Wait()
+	romCache.RLock()
+	romCacheJson, err := json.Marshal(romCache)
+	romCache.RUnlock()
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		log.Printf("failed to get lineageOSROMs: %v", err)
+		log.Printf("failed to marshal romCache to json: %v", err)
 		return
 	}
 
-	httpResponseJSON := &HttpResponseJSON{Response: lineageOSROMs}
+	err = os.WriteFile(cacheFile, romCacheJson, 0644)
+	if err != nil {
+		log.Printf("failed to write '%s' file: %v", cacheFile, err)
+		log.Printf("attempting to remove '%s' to ensure integrity during next program startup...", cacheFile)
+		err = os.Remove(cacheFile)
+		if err != nil {
+			log.Printf("failed to remove file '%s': %v", cacheFile, err)
+		}
+		return
+	}
+}
+
+// http - GET /
+// The LineageOS updater app needs a JSON array of type LineageOSROM
+// Marshal's romCache.ROMs to JSON to serve as the response body
+
+func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
+	go func() { // Also checks for new builds - TBD need a better method as the first request will return no new updates.  inotify?
+		newBuilds := moveBuildArtifacts()
+		if newBuilds {
+			updateROMCache()
+		}
+	}()
+
+	romCache.RLock()
+	httpResponseJSON := &HTTPResponseJSON{Response: romCache.ROMs}
+	romCache.RUnlock()
 
 	b, err := json.Marshal(httpResponseJSON)
 	if err != nil {

processFiles.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"strings"
+)
+
+// Searches the build toolchain output directory for new LineageOS builds.
+// Any new builds are moved into the public http server directory.
+// Returns true if new builds were moved
+func moveBuildArtifacts() bool {
+
+	f, err := os.Open(buildOutDirectory)
+	if err != nil {
+		log.Printf("failed to open ROM directory: %v", err)
+		return false
+	}
+	defer f.Close()
+	files, err := f.ReadDir(0)
+	if err != nil {
+		log.Printf("failed to read files in directory: %v", err)
+		return false
+	}
+
+	var newROMs bool
+
+	for _, v := range files {
+		if isLineageROM, _ := parseROMFileName(v); !isLineageROM { // skip files that aren't LineageOS ROMs
+			continue
+		}
+
+		newROMs = true
+		log.Printf("new build found - moving file %s", v.Name())
+		romCache.Lock() // RW lock to prevent multiple concurrent goroutines moving the same file
+		err := copyThenDeleteFile(fmt.Sprintf("%s/%s", buildOutDirectory, v.Name()), fmt.Sprintf("%s/%s", romDirectory, v.Name()))
+		romCache.Unlock()
+		if err != nil {
+			log.Printf("failed to move file '%s' from out to rom directory: %v", v.Name(), err)
+			continue
+		}
+	}
+
+	return newROMs
+}
+
+// Returns a sha256 hash of a file located at the path provided
+func hashFile(filename string) (string, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err)
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+// false if a file is not a LineageOS ROM .zip file
+// no formal validation is performed - only file naming convention is checked
+// also returns a lineage ROM's filename sliced and delimited by -'s
+// Example filename: lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
+func parseROMFileName(v fs.DirEntry) (bool, []string) {
+
+	// skip directories, non .zip files and files that don't begin with lineage-
+	if v.Type().IsDir() || !strings.HasSuffix(v.Name(), ".zip") || !strings.HasPrefix(v.Name(), "lineage-") {
+		return false, nil
+	}
+
+	splitName := strings.Split(v.Name(), "-")
+	// expect 5 dashes
+	return len(splitName) == 5, splitName
+}
+
+// A custom "move file" function because in docker container the mounted folders are different overlay filesystems
+// Instead of os.Rename, we must copy and delete
+func copyThenDeleteFile(src, dst string) error {
+	sourceFileStat, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", src)
+	}
+
+	source, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+	_, err = io.Copy(destination, source)
+	if err != nil {
+		return fmt.Errorf("failed to copy file: %v", err)
+	}
+
+	err = os.Remove(src)
+	if err != nil {
+		return fmt.Errorf("failed to delete source file after copy: %v", err)
+	}
+
+	return nil
+}