Update extra/product.mk

.drone.yml

@@ -3,7 +3,7 @@ name: default
 
 workspace:
   base: /go
-  path: src/deadbeef.codes/steven/lineageos-ota-server
+  path: src/code.stevenpolley.net/steven/lineageos-ota-server
 
 steps:
 
@@ -22,4 +22,4 @@ steps:
 - name: package in docker container
   image: plugins/docker
   settings:
-    repo: registry.deadbeef.codes/lineageos-ota-server
+    repo: registry.stevenpolley.net/lineageos-ota-server

README.md

@@ -2,4 +2,49 @@
 
 # lineageos-ota-server
 
-OTA Server for LineageOS
+A highly-scalable and lightweight OTA Server for LineageOS.  The OTA server supports a single device model (eg: Google Pixel), if you intend to have OTA's available for a variety of device models, just create an additional OTA server instance for each device.  
+
+![alt text][logo]
+
+[logo]: https://deadbeef.codes/steven/lineageos-ota-server/raw/branch/main/ota-logo.jpg "LineageOS OTA Server"
+
+
+### docker compose example
+
+The service listens on port 8080 by default.  Mount the output directory for the builds of the device you wish to serve to the /out folder.  Also, mount a persistent public directory which will be served publicly.  The public folder can be shared across multiple OTA server instances if you wish.  
+
+```yaml
+
+version: '3.8'
+
+  lineageos-ota:
+    image: registry.deadbeef.codes/lineageos-ota-server:latest
+    restart: always
+    expose:
+      - "8080"
+    volumes:
+      - /data/android/lineage/out/target/product/sunfish:/out
+      - /data/android/public/sunfish:/public
+    
+```
+
+### How to point device to OTA server
+
+The recommended way is to include the configuration inside your build of the ROM.  We do this by by including the URL as a system build.prop by doing the following:
+
+Create new file /data/android/lineage/vendor/lineage/build/core/deadbeef-ota.mk
+
+
+
+Edit (create if not exist) /data/android/lineage/vendor/extra/product.mk
+
+```makefile
+# Include deadbeef.codes OTA server
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        lineage.updater.uri=https://lineageos-ota-{device}.stevenpolley.net
+
+# Default ADB shell
+PRODUCT_SYSTEM_DEFAULT_PROPERTIES += \
+        persist.sys.adb.shell=/system_ext/bin/bash
+
+```

go.mod

@@ -1,3 +1,3 @@
 module deadbeef.codes/steven/lineageos-ota-server
 
-go 1.20
+go 1.22

lineage-android-build-files/README.md

@@ -1,7 +0,0 @@
-These files need to be deployed into your Android source code directory to ensure build.prop file on the device contains the correct URL for receiving OTA updates.  This is to point the updated app at the OTA server you're hosting.
-
-1. Create the directory lineage/vendor/extra
-2. Review and edit the contents of file product.prop to ensure the URL is what you want your device pointed to.
-3. Place the product.mk and product.prop files in this directory
-
-The files will then be automatically read by the build system and patch build.prop with the updated URL.

lineage-android-build-files/product.mk

@@ -1 +0,0 @@
-TARGET_PRODUCT_PROP += $(COMMON_PATH)/product.prop

lineage-android-build-files/product.prop

@@ -1 +0,0 @@
-lineage.updater.uri=https://lineage-ota.deadbeef.codes

main.go

@@ -7,27 +7,77 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path/filepath"
-	"strings"
+	"sync"
 )
 
 // Information for a LineageOS ROM available for download
 type LineageOSROM struct {
-	Datetime int    `json:"datetime"`
-	Filename string `json:"filename"`
-	ID       string `json:"id"`
-	Romtype  string `json:"romtype"`
-	Size     int    `json:"size"`
-	URL      string `json:"url"`
-	Version  string `json:"version"`
+	Datetime int    `json:"datetime"` // Unix timestamp - i.e. 1685907926
+	Filename string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
+	ID       string `json:"id"`       // A unique identifier such as a SHA256 hash of the .zip  - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93
+	Romtype  string `json:"romtype"`  // i.e. NIGHTLY or UNOFFICIAL
+	Size     int    `json:"size"`     // size of .zip file in bytes
+	URL      string `json:"url"`      // Accessible URL where client could download the .zip file
+	Version  string `json:"version"`  // LineageOS version - i.e. 20.0
 }
 
 // The HTTP response JSON should be a JSON array of lineageOSROMS available for download
-type HttpResponseJSON struct {
+type HTTPResponseJSON struct {
 	Response []LineageOSROM `json:"response"`
 }
 
-// HTTP Routing
+// Caches data about available ROMs in memory so we don't need to reference the filesystem for each request
+type ROMCache struct {
+	ROMs         []LineageOSROM  `json:"roms"`
+	Cached       map[string]bool `json:"-"` // to quickly lookup if a file is already cached
+	sync.RWMutex `json:"-"`      // We have multiple goroutines that may be accessing this data simultaneously, so we much lock / unlock it to prevent race conditions
+}
+
+const (
+	romDirectory      = "public"               // directory where ROMs are available for download
+	buildOutDirectory = "out"                  // directory from build system containing artifacts which we can move to romDirectory
+	cacheFile         = "public/romcache.json" // persistence between server restarts so we don't have to rehash all the ROM files each time the program starts
+)
+
+var ( // evil global variables
+	romCache ROMCache
+	baseURL  string
+)
+
+func init() {
+	// intialize and load ROMCache from file - so we don't have to rehash all the big files again
+	romCache = ROMCache{}
+	baseURL = os.Getenv("baseurl")
+	if len(baseURL) < 1 {
+		log.Fatalf("required environment variable 'baseurl' is not set.")
+	}
+
+	romCacheJson, err := os.ReadFile(cacheFile)
+	if err != nil {
+		if err != os.ErrNotExist { // don't care if it doesn't exist, just skip it
+			log.Printf("failed to read romCache file : %v", err)
+		}
+	} else { // if opening the file's successful, then load the contents
+		err = json.Unmarshal(romCacheJson, &romCache)
+		if err != nil {
+			log.Printf("failed to unmarshal romCacheJson to romCache struct: %v", err)
+		}
+
+	}
+
+	romCache.Cached = make(map[string]bool)
+
+	for _, rom := range romCache.ROMs {
+		romCache.Cached[rom.Filename] = true
+		log.Printf("loaded cached file: %s", rom.Filename)
+	}
+
+	// Check if any new build artifacts and load any new files into the romCache
+	moveBuildArtifacts()
+	go updateROMCache()
+}
+
+// HTTP Server
 func main() {
 
 	//Public static files
@@ -38,71 +88,116 @@ func main() {
 
 	log.Print("Service listening on :8080")
 	log.Fatal(http.ListenAndServe(":8080", nil))
-
 }
 
 // Reads the ROM files on the filesystem and populates a slice of linageOSROMs
-func getLineageOSROMs(romDirectory string) ([]LineageOSROM, error) {
+func updateROMCache() {
 
-	if _, err := os.Stat(romDirectory); os.IsNotExist(err) {
-		return nil, fmt.Errorf("romDirectory '%s' does not exist", romDirectory)
-	}
+	log.Printf("updating ROM cache")
 
-	// Get all the zip files and populate romFileNames slice
-	var lineageOSROMs []LineageOSROM
-
-	err := filepath.WalkDir(romDirectory, func(s string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return fmt.Errorf("walk error occured during file '%s': %v", d.Name(), err)
-		}
-		if filepath.Ext(d.Name()) != ".zip" {
-			return nil
-		}
-
-		fInfo, err := d.Info()
-		if err != nil {
-			return fmt.Errorf("failed to get file info '%s': %v", d.Name(), err)
-		}
-
-		// Get information about file and populate rom
-
-		splitName := strings.Split(d.Name(), "-")
-		if len(splitName) != 5 {
-			log.Printf("ignoring zip file '%d', name is not formatted correctly ")
-		}
-
-		lineageOSROM := LineageOSROM{
-			Datetime: int(fInfo.ModTime().Unix()),
-			Filename: d.Name(),
-			ID:       "TBD",
-			Romtype:  "nightly",
-			Size:     int(fInfo.Size()),
-			URL:      fmt.Sprintf("https://lineageos-updater.deadbeef.codes/public/%s", d.Name()),
-			Version:  "TBD",
-		}
-
-		lineageOSROMs = append(lineageOSROMs, lineageOSROM)
-
-		return nil
-	})
+	f, err := os.Open(romDirectory)
 	if err != nil {
-		return nil, fmt.Errorf("failed to walk romDirectory '%s': %v", romDirectory, err)
+		log.Printf("failed to open ROM directory: %v", err)
+		return
 	}
-
-	return lineageOSROMs, nil
-}
-
-// http - GET /
-// Writes JSON response for the updater app to know what versions are available to download
-func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
-	lineageOSROMs, err := getLineageOSROMs("public")
+	defer f.Close()
+	files, err := f.ReadDir(0)
 	if err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-		log.Printf("failed to get lineageOSROMs: %v", err)
+		log.Printf("failed to read files in directory: %v", err)
 		return
 	}
 
-	httpResponseJSON := &HttpResponseJSON{Response: lineageOSROMs}
+	wg := sync.WaitGroup{}
+
+	for i, v := range files {
+
+		isLineageROM, splitName := parseROMFileName(v)
+		if !isLineageROM {
+			continue
+		}
+
+		// skip already cached files
+		romCache.RLock()
+		if _, ok := romCache.Cached[v.Name()]; ok {
+			romCache.RUnlock()
+			continue
+		}
+		romCache.RUnlock()
+
+		wg.Add(1)
+		go func(v fs.DirEntry) {
+			defer wg.Done()
+			fInfo, err := v.Info()
+			if err != nil {
+				log.Printf("failed to get file info '%s': %v", v.Name(), err)
+				return
+			}
+
+			fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, v.Name()))
+			if err != nil {
+				log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", v.Name(), err)
+				return
+			}
+
+			lineageOSROM := LineageOSROM{
+				Datetime: int(fInfo.ModTime().Unix()),
+				Filename: v.Name(),
+				ID:       fileHash,
+				Romtype:  splitName[3], // UNOFFICIAL
+				Size:     int(fInfo.Size()),
+				URL:      fmt.Sprintf("%s/public/%s", baseURL, v.Name()),
+				Version:  splitName[1],
+			}
+
+			romCache.Lock()
+			if _, ok := romCache.Cached[v.Name()]; ok { // it's possible another goroutine was already working to cache the file, so we check at the end
+				romCache.Unlock()
+				return
+			}
+			romCache.ROMs = append(romCache.ROMs, lineageOSROM)
+			romCache.Cached[v.Name()] = true
+			romCache.Unlock()
+
+		}(files[i])
+	}
+
+	// save file to disk for next startup so we don't have to rehash all the files again
+	wg.Wait()
+	romCache.RLock()
+	romCacheJson, err := json.Marshal(romCache)
+	romCache.RUnlock()
+	if err != nil {
+		log.Printf("failed to marshal romCache to json: %v", err)
+		return
+	}
+
+	err = os.WriteFile(cacheFile, romCacheJson, 0644)
+	if err != nil {
+		log.Printf("failed to write '%s' file: %v", cacheFile, err)
+		log.Printf("attempting to remove '%s' to ensure integrity during next program startup...", cacheFile)
+		err = os.Remove(cacheFile)
+		if err != nil {
+			log.Printf("failed to remove file '%s': %v", cacheFile, err)
+		}
+		return
+	}
+}
+
+// http - GET /
+// The LineageOS updater app needs a JSON array of type LineageOSROM
+// Marshal's romCache.ROMs to JSON to serve as the response body
+
+func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
+	go func() { // Also checks for new builds - TBD need a better method as the first request will return no new updates.  inotify?
+		newBuilds := moveBuildArtifacts()
+		if newBuilds {
+			updateROMCache()
+		}
+	}()
+
+	romCache.RLock()
+	httpResponseJSON := &HTTPResponseJSON{Response: romCache.ROMs}
+	romCache.RUnlock()
 
 	b, err := json.Marshal(httpResponseJSON)
 	if err != nil {

processFiles.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"strings"
+)
+
+// Searches the build toolchain output directory for new LineageOS builds.
+// Any new builds are moved into the public http server directory.
+// Returns true if new builds were moved
+func moveBuildArtifacts() bool {
+
+	f, err := os.Open(buildOutDirectory)
+	if err != nil {
+		log.Printf("failed to open ROM directory: %v", err)
+		return false
+	}
+	defer f.Close()
+	files, err := f.ReadDir(0)
+	if err != nil {
+		log.Printf("failed to read files in directory: %v", err)
+		return false
+	}
+
+	var newROMs bool
+
+	for _, v := range files {
+		if isLineageROM, _ := parseROMFileName(v); !isLineageROM { // skip files that aren't LineageOS ROMs
+			continue
+		}
+
+		newROMs = true
+		log.Printf("new build found - moving file %s", v.Name())
+		romCache.Lock() // RW lock to prevent multiple concurrent goroutines moving the same file
+		err := copyThenDeleteFile(fmt.Sprintf("%s/%s", buildOutDirectory, v.Name()), fmt.Sprintf("%s/%s", romDirectory, v.Name()))
+		romCache.Unlock()
+		if err != nil {
+			log.Printf("failed to move file '%s' from out to rom directory: %v", v.Name(), err)
+			continue
+		}
+	}
+
+	return newROMs
+}
+
+// Returns a sha256 hash of a file located at the path provided
+func hashFile(filename string) (string, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err)
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+// false if a file is not a LineageOS ROM .zip file
+// no formal validation is performed - only file naming convention is checked
+// also returns a lineage ROM's filename sliced and delimited by -'s
+// Example filename: lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
+func parseROMFileName(v fs.DirEntry) (bool, []string) {
+
+	// skip directories, non .zip files and files that don't begin with lineage-
+	if v.Type().IsDir() || !strings.HasSuffix(v.Name(), ".zip") || !strings.HasPrefix(v.Name(), "lineage-") {
+		return false, nil
+	}
+
+	splitName := strings.Split(v.Name(), "-")
+	// expect 5 dashes
+	return len(splitName) == 5, splitName
+}
+
+// A custom "move file" function because in docker container the mounted folders are different overlay filesystems
+// Instead of os.Rename, we must copy and delete
+func copyThenDeleteFile(src, dst string) error {
+	sourceFileStat, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", src)
+	}
+
+	source, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destination.Close()
+	_, err = io.Copy(destination, source)
+	if err != nil {
+		return fmt.Errorf("failed to copy file: %v", err)
+	}
+
+	err = os.Remove(src)
+	if err != nil {
+		return fmt.Errorf("failed to delete source file after copy: %v", err)
+	}
+
+	return nil
+}