add v2 ota api for lineageos

main.go

@@ -7,18 +7,52 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
+	"strings"
 	"sync"
+	"time"
 )
 
 // Information for a LineageOS ROM available for download
 type LineageOSROM struct {
-	Datetime int    `json:"datetime"` // Unix timestamp - i.e. 1685907926
+	Datetime         int    `json:"datetime"` // Unix timestamp - i.e. 1685907926
-	Filename string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
+	Filename         string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip
-	ID       string `json:"id"`       // A unique identifier such as a SHA256 hash of the .zip  - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93
+	ID               string `json:"id"`       // A unique identifier such as a SHA256 hash of the .zip  - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93
-	Romtype  string `json:"romtype"`  // i.e. NIGHTLY or UNOFFICIAL
+	Romtype          string `json:"romtype"`  // i.e. NIGHTLY or UNOFFICIAL
-	Size     int    `json:"size"`     // size of .zip file in bytes
+	Size             int    `json:"size"`     // size of .zip file in bytes
-	URL      string `json:"url"`      // Accessible URL where client could download the .zip file
+	URL              string `json:"url"`      // Accessible URL where client could download the .zip file
-	Version  string `json:"version"`  // LineageOS version - i.e. 20.0
+	Version          string `json:"version"`  // LineageOS version - i.e. 20.0
+	// v2 API fields
+	SHA1             string `json:"sha1,omitempty"`
+	SHA256           string `json:"sha256,omitempty"`
+	Date             string `json:"date,omitempty"`
+	OSPatchLevel     string `json:"os_patch_level,omitempty"`
+	OSSDKLevel       int    `json:"os_sdk_level,omitempty"`
+	OTAPropertyFiles string `json:"ota_property_files,omitempty"`
+}
+
+// Structs for v2 API responses
+type V2BuildResponse struct {
+	Date     string        `json:"date"`
+	Datetime int64         `json:"datetime"`
+	Files    []V2BuildFile `json:"files"`
+	Type     string        `json:"type"`
+	Version  string        `json:"version"`
+}
+
+type V2BuildFile struct {
+	Date             string `json:"date,omitempty"`
+	Datetime         int64  `json:"datetime,omitempty"`
+	Filename         string `json:"filename"`
+	Filepath         string `json:"filepath"`
+	OSPatchLevel     string `json:"os_patch_level,omitempty"`
+	OSSDKLevel       int    `json:"os_sdk_level,omitempty"`
+	OTAPropertyFiles string `json:"ota_property_files,omitempty"`
+	SHA1             string `json:"sha1"`
+	SHA256           string `json:"sha256"`
+	Size             int64  `json:"size"`
+	Type             string `json:"type,omitempty"`
+	URL              string `json:"url"`
 }
 
 // The HTTP response JSON should be a JSON array of lineageOSROMS available for download
@@ -30,7 +64,7 @@ type HTTPResponseJSON struct {
 type ROMCache struct {
 	ROMs         []LineageOSROM  `json:"roms"`
 	Cached       map[string]bool `json:"-"` // to quickly lookup if a file is already cached
-	sync.RWMutex `json:"-"`      // We have multiple goroutines that may be accessing this data simultaneously, so we much lock / unlock it to prevent race conditions
+	sync.RWMutex `json:"-"`      // We have multiple goroutines that may be accessing this data simultaneously, so we must lock / unlock it to prevent race conditions
 }
 
 const (
@@ -62,15 +96,22 @@ func init() {
 		if err != nil {
 			log.Printf("failed to unmarshal romCacheJson to romCache struct: %v", err)
 		}
-
 	}
 
 	romCache.Cached = make(map[string]bool)
 
+	var validROMs []LineageOSROM
 	for _, rom := range romCache.ROMs {
-		romCache.Cached[rom.Filename] = true
+		// A ROM is valid for v2 if it has SHA1 and Date fields populated
-		log.Printf("loaded cached file: %s", rom.Filename)
+		if rom.SHA1 != "" && rom.Date != "" {
+			romCache.Cached[rom.Filename] = true
+			validROMs = append(validROMs, rom)
+			log.Printf("loaded cached file: %s", rom.Filename)
+		} else {
+			log.Printf("cache entry for %s is missing v2 metadata, will re-process", rom.Filename)
+		}
 	}
+	romCache.ROMs = validROMs
 
 	// Check if any new build artifacts and load any new files into the romCache
 	moveBuildArtifacts()
@@ -83,82 +124,129 @@ func main() {
 	//Public static files
 	http.Handle("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir("public"))))
 
-	// ROM list
+	// ROM list (legacy / catch-all)
 	http.HandleFunc("/", lineageOSROMListHandler)
 
+	// V2 ROM list and device info
+	http.HandleFunc("/api/v2/devices/", lineageOSROMListHandlerV2)
+
 	log.Print("Service listening on :8080")
 	log.Fatal(http.ListenAndServe(":8080", nil))
 }
 
-// Reads the ROM files on the filesystem and populates a slice of linageOSROMs
+// Reads the ROM files on the filesystem recursively and populates a slice of lineageOSROMs
 func updateROMCache() {
 
 	log.Printf("updating ROM cache")
 
-	f, err := os.Open(romDirectory)
+	type fileInfoPath struct {
-	if err != nil {
+		path      string
-		log.Printf("failed to open ROM directory: %v", err)
+		info      fs.DirEntry
-		return
+		splitName []string
 	}
-	defer f.Close()
+
-	files, err := f.ReadDir(0)
+	var romFiles []fileInfoPath
+
+	err := filepath.WalkDir(romDirectory, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		isLineageROM, splitName := parseROMFileName(d)
+		if isLineageROM {
+			romFiles = append(romFiles, fileInfoPath{
+				path:      path,
+				info:      d,
+				splitName: splitName,
+			})
+		}
+		return nil
+	})
+
 	if err != nil {
-		log.Printf("failed to read files in directory: %v", err)
+		log.Printf("failed to walk ROM directory: %v", err)
 		return
 	}
 
 	wg := sync.WaitGroup{}
 
-	for i, v := range files {
+	for i := range romFiles {
-
+		v := romFiles[i]
-		isLineageROM, splitName := parseROMFileName(v)
-		if !isLineageROM {
-			continue
-		}
 
 		// skip already cached files
 		romCache.RLock()
-		if _, ok := romCache.Cached[v.Name()]; ok {
+		if _, ok := romCache.Cached[v.info.Name()]; ok {
 			romCache.RUnlock()
 			continue
 		}
 		romCache.RUnlock()
 
 		wg.Add(1)
-		go func(v fs.DirEntry) {
+		go func(file fileInfoPath) {
 			defer wg.Done()
-			fInfo, err := v.Info()
+			fInfo, err := file.info.Info()
 			if err != nil {
-				log.Printf("failed to get file info '%s': %v", v.Name(), err)
+				log.Printf("failed to get file info '%s': %v", file.info.Name(), err)
 				return
 			}
 
-			fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, v.Name()))
+			sha1Val, sha256Val, err := hashFileSha1AndSha256(file.path)
 			if err != nil {
-				log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", v.Name(), err)
+				log.Printf("ignore zip file '%s', failed to calculate hashes: %v", file.info.Name(), err)
 				return
 			}
 
+			// Read ZIP metadata
+			meta, err := readZipMetadata(file.path)
+			if err != nil {
+				log.Printf("failed to read metadata inside zip '%s': %v", file.info.Name(), err)
+			}
+
+			// Determine timestamp
+			timestamp := int(fInfo.ModTime().Unix())
+			if meta.Timestamp > 0 {
+				timestamp = meta.Timestamp
+			}
+
+			// Format build date as YYYY-MM-DD
+			buildDate := time.Unix(int64(timestamp), 0).UTC().Format("2006-01-02")
+
+			// Calculate URL relative path from public directory
+			relPath, err := filepath.Rel(romDirectory, file.path)
+			if err != nil {
+				log.Printf("failed to calculate relative path for '%s': %v", file.path, err)
+				relPath = file.info.Name()
+			}
+			relPathSlash := filepath.ToSlash(relPath)
+
 			lineageOSROM := LineageOSROM{
-				Datetime: int(fInfo.ModTime().Unix()),
+				Datetime:         timestamp,
-				Filename: v.Name(),
+				Filename:         file.info.Name(),
-				ID:       fileHash,
+				ID:               sha256Val, // Keep ID as SHA256 for backward compatibility
-				Romtype:  splitName[3], // UNOFFICIAL
+				Romtype:          file.splitName[3],
-				Size:     int(fInfo.Size()),
+				Size:             int(fInfo.Size()),
-				URL:      fmt.Sprintf("%s/public/%s", baseURL, v.Name()),
+				URL:              fmt.Sprintf("%s/public/%s", baseURL, relPathSlash),
-				Version:  splitName[1],
+				Version:          file.splitName[1],
+				SHA1:             sha1Val,
+				SHA256:           sha256Val,
+				Date:             buildDate,
+				OSPatchLevel:     meta.PatchLevel,
+				OSSDKLevel:       meta.SDKLevel,
+				OTAPropertyFiles: meta.OTAPropertyFiles,
 			}
 
 			romCache.Lock()
-			if _, ok := romCache.Cached[v.Name()]; ok { // it's possible another goroutine was already working to cache the file, so we check at the end
+			if _, ok := romCache.Cached[file.info.Name()]; ok { // it's possible another goroutine was already working to cache the file, so we check at the end
 				romCache.Unlock()
 				return
 			}
 			romCache.ROMs = append(romCache.ROMs, lineageOSROM)
-			romCache.Cached[v.Name()] = true
+			romCache.Cached[file.info.Name()] = true
 			romCache.Unlock()
 
-		}(files[i])
+		}(v)
 	}
 
 	// save file to disk for next startup so we don't have to rehash all the files again
@@ -185,15 +273,9 @@ func updateROMCache() {
 
 // http - GET /
 // The LineageOS updater app needs a JSON array of type LineageOSROM
-// Marshal's romCache.ROMs to JSON to serve as the response body
+// Marshals romCache.ROMs to JSON to serve as the response body
-
 func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
-	go func() { // Also checks for new builds - TBD need a better method as the first request will return no new updates.  inotify?
+	checkNewBuilds()
-		newBuilds := moveBuildArtifacts()
-		if newBuilds {
-			updateROMCache()
-		}
-	}()
 
 	romCache.RLock()
 	httpResponseJSON := &HTTPResponseJSON{Response: romCache.ROMs}
@@ -208,3 +290,135 @@ func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
 
 	w.Write(b)
 }
+
+func checkNewBuilds() {
+	go func() { // Also checks for new builds
+		newBuilds := moveBuildArtifacts()
+		if newBuilds {
+			updateROMCache()
+		}
+	}()
+}
+
+func lineageOSROMListHandlerV2(w http.ResponseWriter, r *http.Request) {
+	checkNewBuilds()
+
+	// Path: /api/v2/devices/{device}/builds
+	// or: /api/v2/devices/{device}
+	parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
+	if len(parts) < 4 {
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	device := parts[3]
+	isBuildsRequest := len(parts) >= 5 && parts[4] == "builds"
+
+	if isBuildsRequest {
+		romCache.RLock()
+		defer romCache.RUnlock()
+
+		var builds []V2BuildResponse
+
+		for _, rom := range romCache.ROMs {
+			filenameParts := strings.Split(strings.TrimSuffix(rom.Filename, ".zip"), "-")
+			romDevice := ""
+			if len(filenameParts) > 0 {
+				romDevice = filenameParts[len(filenameParts)-1]
+			}
+
+			if romDevice != device {
+				continue
+			}
+
+			relPath := rom.Filename
+			if strings.Contains(rom.URL, "/public/") {
+				parts := strings.SplitN(rom.URL, "/public/", 2)
+				if len(parts) == 2 {
+					relPath = parts[1]
+				}
+			}
+
+			file := V2BuildFile{
+				Date:             rom.Date,
+				Datetime:         int64(rom.Datetime),
+				Filename:         rom.Filename,
+				Filepath:         "/public/" + relPath,
+				OSPatchLevel:     rom.OSPatchLevel,
+				OSSDKLevel:       rom.OSSDKLevel,
+				OTAPropertyFiles: rom.OTAPropertyFiles,
+				SHA1:             rom.SHA1,
+				SHA256:           rom.SHA256,
+				Size:             int64(rom.Size),
+				Type:             strings.ToLower(rom.Romtype),
+				URL:              rom.URL,
+			}
+
+			build := V2BuildResponse{
+				Date:     rom.Date,
+				Datetime: int64(rom.Datetime),
+				Files:    []V2BuildFile{file},
+				Type:     strings.ToLower(rom.Romtype),
+				Version:  rom.Version,
+			}
+
+			builds = append(builds, build)
+		}
+
+		if builds == nil {
+			builds = []V2BuildResponse{}
+		}
+
+		b, err := json.Marshal(builds)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			log.Printf("failed to marshal v2 builds to json: %v", err)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(b)
+	} else {
+		romCache.RLock()
+		defer romCache.RUnlock()
+
+		versionsSet := make(map[string]bool)
+		for _, rom := range romCache.ROMs {
+			filenameParts := strings.Split(strings.TrimSuffix(rom.Filename, ".zip"), "-")
+			romDevice := ""
+			if len(filenameParts) > 0 {
+				romDevice = filenameParts[len(filenameParts)-1]
+			}
+			if romDevice == device {
+				versionsSet[rom.Version] = true
+			}
+		}
+
+		var versions []string
+		for v := range versionsSet {
+			versions = append(versions, v)
+		}
+		if versions == nil {
+			versions = []string{}
+		}
+
+		deviceInfo := map[string]interface{}{
+			"name":         device,
+			"model":        device,
+			"oem":          "LineageOS",
+			"info_url":     fmt.Sprintf("https://wiki.lineageos.org/devices/%s/", device),
+			"versions":     versions,
+			"dependencies": []string{},
+		}
+
+		b, err := json.Marshal(deviceInfo)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			log.Printf("failed to marshal v2 device info to json: %v", err)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(b)
+	}
+}

processFiles.go

@@ -1,15 +1,26 @@
 package main
 
 import (
+	"archive/zip"
+	"bufio"
+	"crypto/sha1"
 	"crypto/sha256"
 	"fmt"
 	"io"
 	"io/fs"
 	"log"
 	"os"
+	"strconv"
 	"strings"
 )
 
+type ZipMetadata struct {
+	Timestamp        int
+	SDKLevel         int
+	PatchLevel       string
+	OTAPropertyFiles string
+}
+
 // Searches the build toolchain output directory for new LineageOS builds.
 // Any new builds are moved into the public http server directory.
 // Returns true if new builds were moved
@@ -50,18 +61,83 @@ func moveBuildArtifacts() bool {
 
 // Returns a sha256 hash of a file located at the path provided
 func hashFile(filename string) (string, error) {
+	_, sha256Val, err := hashFileSha1AndSha256(filename)
+	return sha256Val, err
+}
+
+// Returns sha1 and sha256 hashes of a file located at the path provided
+func hashFileSha1AndSha256(filename string) (string, string, error) {
 	f, err := os.Open(filename)
 	if err != nil {
-		return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err)
+		return "", "", fmt.Errorf("failed to open file '%s': %v", filename, err)
 	}
 	defer f.Close()
 
-	h := sha256.New()
+	h1 := sha1.New()
-	if _, err := io.Copy(h, f); err != nil {
+	h256 := sha256.New()
-		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
+
+	buf := make([]byte, 32768)
+	for {
+		n, err := f.Read(buf)
+		if n > 0 {
+			h1.Write(buf[:n])
+			h256.Write(buf[:n])
+		}
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", "", fmt.Errorf("failed to read file '%s': %v", filename, err)
+		}
 	}
 
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
+	return fmt.Sprintf("%x", h1.Sum(nil)), fmt.Sprintf("%x", h256.Sum(nil)), nil
+}
+
+// Reads metadata from META-INF/com/android/metadata inside the zip file
+func readZipMetadata(zipPath string) (ZipMetadata, error) {
+	var meta ZipMetadata
+
+	r, err := zip.OpenReader(zipPath)
+	if err != nil {
+		return meta, err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+		if f.Name == "META-INF/com/android/metadata" {
+			rc, err := f.Open()
+			if err != nil {
+				return meta, err
+			}
+			defer rc.Close()
+
+			scanner := bufio.NewScanner(rc)
+			for scanner.Scan() {
+				line := scanner.Text()
+				parts := strings.SplitN(line, "=", 2)
+				if len(parts) == 2 {
+					key := strings.TrimSpace(parts[0])
+					val := strings.TrimSpace(parts[1])
+					switch key {
+					case "post-timestamp":
+						t, _ := strconv.Atoi(val)
+						meta.Timestamp = t
+					case "post-sdk-level":
+						s, _ := strconv.Atoi(val)
+						meta.SDKLevel = s
+					case "security-patch-level":
+						meta.PatchLevel = val
+					case "ota-property-files":
+						meta.OTAPropertyFiles = val
+					}
+				}
+			}
+			return meta, scanner.Err()
+		}
+	}
+
+	return meta, fmt.Errorf("metadata file not found in zip")
 }
 
 // false if a file is not a LineageOS ROM .zip file