diff --git a/main.go b/main.go index f91ee4d..0bb8104 100644 --- a/main.go +++ b/main.go @@ -7,18 +7,52 @@ import ( "log" "net/http" "os" + "path/filepath" + "strings" "sync" + "time" ) // Information for a LineageOS ROM available for download type LineageOSROM struct { - Datetime int `json:"datetime"` // Unix timestamp - i.e. 1685907926 - Filename string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip - ID string `json:"id"` // A unique identifier such as a SHA256 hash of the .zip - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93 - Romtype string `json:"romtype"` // i.e. NIGHTLY or UNOFFICIAL - Size int `json:"size"` // size of .zip file in bytes - URL string `json:"url"` // Accessible URL where client could download the .zip file - Version string `json:"version"` // LineageOS version - i.e. 20.0 + Datetime int `json:"datetime"` // Unix timestamp - i.e. 1685907926 + Filename string `json:"filename"` // .zip filename - i.e. lineage-20.0-20230604-UNOFFICIAL-sunfish.zip + ID string `json:"id"` // A unique identifier such as a SHA256 hash of the .zip - i.e. 603bfc02e403e5fd1bf9ed74383f1d6c9ec7fb228d03c4b37753033d79488e93 + Romtype string `json:"romtype"` // i.e. NIGHTLY or UNOFFICIAL + Size int `json:"size"` // size of .zip file in bytes + URL string `json:"url"` // Accessible URL where client could download the .zip file + Version string `json:"version"` // LineageOS version - i.e. 20.0 + // v2 API fields + SHA1 string `json:"sha1,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Date string `json:"date,omitempty"` + OSPatchLevel string `json:"os_patch_level,omitempty"` + OSSDKLevel int `json:"os_sdk_level,omitempty"` + OTAPropertyFiles string `json:"ota_property_files,omitempty"` +} + +// Structs for v2 API responses +type V2BuildResponse struct { + Date string `json:"date"` + Datetime int64 `json:"datetime"` + Files []V2BuildFile `json:"files"` + Type string `json:"type"` + Version string `json:"version"` +} + +type V2BuildFile struct { + Date string `json:"date,omitempty"` + Datetime int64 `json:"datetime,omitempty"` + Filename string `json:"filename"` + Filepath string `json:"filepath"` + OSPatchLevel string `json:"os_patch_level,omitempty"` + OSSDKLevel int `json:"os_sdk_level,omitempty"` + OTAPropertyFiles string `json:"ota_property_files,omitempty"` + SHA1 string `json:"sha1"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Type string `json:"type,omitempty"` + URL string `json:"url"` } // The HTTP response JSON should be a JSON array of lineageOSROMS available for download @@ -30,7 +64,7 @@ type HTTPResponseJSON struct { type ROMCache struct { ROMs []LineageOSROM `json:"roms"` Cached map[string]bool `json:"-"` // to quickly lookup if a file is already cached - sync.RWMutex `json:"-"` // We have multiple goroutines that may be accessing this data simultaneously, so we much lock / unlock it to prevent race conditions + sync.RWMutex `json:"-"` // We have multiple goroutines that may be accessing this data simultaneously, so we must lock / unlock it to prevent race conditions } const ( @@ -62,15 +96,22 @@ func init() { if err != nil { log.Printf("failed to unmarshal romCacheJson to romCache struct: %v", err) } - } romCache.Cached = make(map[string]bool) + var validROMs []LineageOSROM for _, rom := range romCache.ROMs { - romCache.Cached[rom.Filename] = true - log.Printf("loaded cached file: %s", rom.Filename) + // A ROM is valid for v2 if it has SHA1 and Date fields populated + if rom.SHA1 != "" && rom.Date != "" { + romCache.Cached[rom.Filename] = true + validROMs = append(validROMs, rom) + log.Printf("loaded cached file: %s", rom.Filename) + } else { + log.Printf("cache entry for %s is missing v2 metadata, will re-process", rom.Filename) + } } + romCache.ROMs = validROMs // Check if any new build artifacts and load any new files into the romCache moveBuildArtifacts() @@ -83,82 +124,129 @@ func main() { //Public static files http.Handle("/public/", http.StripPrefix("/public/", http.FileServer(http.Dir("public")))) - // ROM list + // ROM list (legacy / catch-all) http.HandleFunc("/", lineageOSROMListHandler) + // V2 ROM list and device info + http.HandleFunc("/api/v2/devices/", lineageOSROMListHandlerV2) + log.Print("Service listening on :8080") log.Fatal(http.ListenAndServe(":8080", nil)) } -// Reads the ROM files on the filesystem and populates a slice of linageOSROMs +// Reads the ROM files on the filesystem recursively and populates a slice of lineageOSROMs func updateROMCache() { log.Printf("updating ROM cache") - f, err := os.Open(romDirectory) - if err != nil { - log.Printf("failed to open ROM directory: %v", err) - return + type fileInfoPath struct { + path string + info fs.DirEntry + splitName []string } - defer f.Close() - files, err := f.ReadDir(0) + + var romFiles []fileInfoPath + + err := filepath.WalkDir(romDirectory, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + isLineageROM, splitName := parseROMFileName(d) + if isLineageROM { + romFiles = append(romFiles, fileInfoPath{ + path: path, + info: d, + splitName: splitName, + }) + } + return nil + }) + if err != nil { - log.Printf("failed to read files in directory: %v", err) + log.Printf("failed to walk ROM directory: %v", err) return } wg := sync.WaitGroup{} - for i, v := range files { - - isLineageROM, splitName := parseROMFileName(v) - if !isLineageROM { - continue - } + for i := range romFiles { + v := romFiles[i] // skip already cached files romCache.RLock() - if _, ok := romCache.Cached[v.Name()]; ok { + if _, ok := romCache.Cached[v.info.Name()]; ok { romCache.RUnlock() continue } romCache.RUnlock() wg.Add(1) - go func(v fs.DirEntry) { + go func(file fileInfoPath) { defer wg.Done() - fInfo, err := v.Info() + fInfo, err := file.info.Info() if err != nil { - log.Printf("failed to get file info '%s': %v", v.Name(), err) + log.Printf("failed to get file info '%s': %v", file.info.Name(), err) return } - fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, v.Name())) + sha1Val, sha256Val, err := hashFileSha1AndSha256(file.path) if err != nil { - log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", v.Name(), err) + log.Printf("ignore zip file '%s', failed to calculate hashes: %v", file.info.Name(), err) return } + // Read ZIP metadata + meta, err := readZipMetadata(file.path) + if err != nil { + log.Printf("failed to read metadata inside zip '%s': %v", file.info.Name(), err) + } + + // Determine timestamp + timestamp := int(fInfo.ModTime().Unix()) + if meta.Timestamp > 0 { + timestamp = meta.Timestamp + } + + // Format build date as YYYY-MM-DD + buildDate := time.Unix(int64(timestamp), 0).UTC().Format("2006-01-02") + + // Calculate URL relative path from public directory + relPath, err := filepath.Rel(romDirectory, file.path) + if err != nil { + log.Printf("failed to calculate relative path for '%s': %v", file.path, err) + relPath = file.info.Name() + } + relPathSlash := filepath.ToSlash(relPath) + lineageOSROM := LineageOSROM{ - Datetime: int(fInfo.ModTime().Unix()), - Filename: v.Name(), - ID: fileHash, - Romtype: splitName[3], // UNOFFICIAL - Size: int(fInfo.Size()), - URL: fmt.Sprintf("%s/public/%s", baseURL, v.Name()), - Version: splitName[1], + Datetime: timestamp, + Filename: file.info.Name(), + ID: sha256Val, // Keep ID as SHA256 for backward compatibility + Romtype: file.splitName[3], + Size: int(fInfo.Size()), + URL: fmt.Sprintf("%s/public/%s", baseURL, relPathSlash), + Version: file.splitName[1], + SHA1: sha1Val, + SHA256: sha256Val, + Date: buildDate, + OSPatchLevel: meta.PatchLevel, + OSSDKLevel: meta.SDKLevel, + OTAPropertyFiles: meta.OTAPropertyFiles, } romCache.Lock() - if _, ok := romCache.Cached[v.Name()]; ok { // it's possible another goroutine was already working to cache the file, so we check at the end + if _, ok := romCache.Cached[file.info.Name()]; ok { // it's possible another goroutine was already working to cache the file, so we check at the end romCache.Unlock() return } romCache.ROMs = append(romCache.ROMs, lineageOSROM) - romCache.Cached[v.Name()] = true + romCache.Cached[file.info.Name()] = true romCache.Unlock() - }(files[i]) + }(v) } // save file to disk for next startup so we don't have to rehash all the files again @@ -185,15 +273,9 @@ func updateROMCache() { // http - GET / // The LineageOS updater app needs a JSON array of type LineageOSROM -// Marshal's romCache.ROMs to JSON to serve as the response body - +// Marshals romCache.ROMs to JSON to serve as the response body func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) { - go func() { // Also checks for new builds - TBD need a better method as the first request will return no new updates. inotify? - newBuilds := moveBuildArtifacts() - if newBuilds { - updateROMCache() - } - }() + checkNewBuilds() romCache.RLock() httpResponseJSON := &HTTPResponseJSON{Response: romCache.ROMs} @@ -208,3 +290,135 @@ func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) { w.Write(b) } + +func checkNewBuilds() { + go func() { // Also checks for new builds + newBuilds := moveBuildArtifacts() + if newBuilds { + updateROMCache() + } + }() +} + +func lineageOSROMListHandlerV2(w http.ResponseWriter, r *http.Request) { + checkNewBuilds() + + // Path: /api/v2/devices/{device}/builds + // or: /api/v2/devices/{device} + parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(parts) < 4 { + http.Error(w, "Bad Request", http.StatusBadRequest) + return + } + + device := parts[3] + isBuildsRequest := len(parts) >= 5 && parts[4] == "builds" + + if isBuildsRequest { + romCache.RLock() + defer romCache.RUnlock() + + var builds []V2BuildResponse + + for _, rom := range romCache.ROMs { + filenameParts := strings.Split(strings.TrimSuffix(rom.Filename, ".zip"), "-") + romDevice := "" + if len(filenameParts) > 0 { + romDevice = filenameParts[len(filenameParts)-1] + } + + if romDevice != device { + continue + } + + relPath := rom.Filename + if strings.Contains(rom.URL, "/public/") { + parts := strings.SplitN(rom.URL, "/public/", 2) + if len(parts) == 2 { + relPath = parts[1] + } + } + + file := V2BuildFile{ + Date: rom.Date, + Datetime: int64(rom.Datetime), + Filename: rom.Filename, + Filepath: "/public/" + relPath, + OSPatchLevel: rom.OSPatchLevel, + OSSDKLevel: rom.OSSDKLevel, + OTAPropertyFiles: rom.OTAPropertyFiles, + SHA1: rom.SHA1, + SHA256: rom.SHA256, + Size: int64(rom.Size), + Type: strings.ToLower(rom.Romtype), + URL: rom.URL, + } + + build := V2BuildResponse{ + Date: rom.Date, + Datetime: int64(rom.Datetime), + Files: []V2BuildFile{file}, + Type: strings.ToLower(rom.Romtype), + Version: rom.Version, + } + + builds = append(builds, build) + } + + if builds == nil { + builds = []V2BuildResponse{} + } + + b, err := json.Marshal(builds) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + log.Printf("failed to marshal v2 builds to json: %v", err) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(b) + } else { + romCache.RLock() + defer romCache.RUnlock() + + versionsSet := make(map[string]bool) + for _, rom := range romCache.ROMs { + filenameParts := strings.Split(strings.TrimSuffix(rom.Filename, ".zip"), "-") + romDevice := "" + if len(filenameParts) > 0 { + romDevice = filenameParts[len(filenameParts)-1] + } + if romDevice == device { + versionsSet[rom.Version] = true + } + } + + var versions []string + for v := range versionsSet { + versions = append(versions, v) + } + if versions == nil { + versions = []string{} + } + + deviceInfo := map[string]interface{}{ + "name": device, + "model": device, + "oem": "LineageOS", + "info_url": fmt.Sprintf("https://wiki.lineageos.org/devices/%s/", device), + "versions": versions, + "dependencies": []string{}, + } + + b, err := json.Marshal(deviceInfo) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + log.Printf("failed to marshal v2 device info to json: %v", err) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(b) + } +} diff --git a/processFiles.go b/processFiles.go index ff01f65..df3848d 100644 --- a/processFiles.go +++ b/processFiles.go @@ -1,15 +1,26 @@ package main import ( + "archive/zip" + "bufio" + "crypto/sha1" "crypto/sha256" "fmt" "io" "io/fs" "log" "os" + "strconv" "strings" ) +type ZipMetadata struct { + Timestamp int + SDKLevel int + PatchLevel string + OTAPropertyFiles string +} + // Searches the build toolchain output directory for new LineageOS builds. // Any new builds are moved into the public http server directory. // Returns true if new builds were moved @@ -50,18 +61,83 @@ func moveBuildArtifacts() bool { // Returns a sha256 hash of a file located at the path provided func hashFile(filename string) (string, error) { + _, sha256Val, err := hashFileSha1AndSha256(filename) + return sha256Val, err +} + +// Returns sha1 and sha256 hashes of a file located at the path provided +func hashFileSha1AndSha256(filename string) (string, string, error) { f, err := os.Open(filename) if err != nil { - return "", fmt.Errorf("failed to open file '%s': %v: ", filename, err) + return "", "", fmt.Errorf("failed to open file '%s': %v", filename, err) } defer f.Close() - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return "", fmt.Errorf("failed to copy data from file to hash function: %v", err) + h1 := sha1.New() + h256 := sha256.New() + + buf := make([]byte, 32768) + for { + n, err := f.Read(buf) + if n > 0 { + h1.Write(buf[:n]) + h256.Write(buf[:n]) + } + if err == io.EOF { + break + } + if err != nil { + return "", "", fmt.Errorf("failed to read file '%s': %v", filename, err) + } } - return fmt.Sprintf("%x", h.Sum(nil)), nil + return fmt.Sprintf("%x", h1.Sum(nil)), fmt.Sprintf("%x", h256.Sum(nil)), nil +} + +// Reads metadata from META-INF/com/android/metadata inside the zip file +func readZipMetadata(zipPath string) (ZipMetadata, error) { + var meta ZipMetadata + + r, err := zip.OpenReader(zipPath) + if err != nil { + return meta, err + } + defer r.Close() + + for _, f := range r.File { + if f.Name == "META-INF/com/android/metadata" { + rc, err := f.Open() + if err != nil { + return meta, err + } + defer rc.Close() + + scanner := bufio.NewScanner(rc) + for scanner.Scan() { + line := scanner.Text() + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + val := strings.TrimSpace(parts[1]) + switch key { + case "post-timestamp": + t, _ := strconv.Atoi(val) + meta.Timestamp = t + case "post-sdk-level": + s, _ := strconv.Atoi(val) + meta.SDKLevel = s + case "security-patch-level": + meta.PatchLevel = val + case "ota-property-files": + meta.OTAPropertyFiles = val + } + } + } + return meta, scanner.Err() + } + } + + return meta, fmt.Errorf("metadata file not found in zip") } // false if a file is not a LineageOS ROM .zip file