diff --git a/dev.sh b/dev.sh index a116ef50..b8beda0c 100755 --- a/dev.sh +++ b/dev.sh @@ -11,7 +11,8 @@ export RM_SMTP_NOTLS=1 export RM_SMTP_NOAUTH=1 export JWT_SECRET_KEY=dev export LOGLEVEL=${1:-DEBUG} -export STORAGE_URL=http://$(hostname):3000 +#export STORAGE_URL=http://$(hostname):3000 +#export STORAGE_URL=http://pc.lan:3000 make runui & PID=$! trap "kill $PID ||:" EXIT diff --git a/go.mod b/go.mod index c8c52b05..2a5a73d2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ddvk/rmfakecloud -go 1.17 +go 1.23 require ( github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 diff --git a/internal/app/handlers.go b/internal/app/handlers.go index 738ce590..82b18558 100644 --- a/internal/app/handlers.go +++ b/internal/app/handlers.go @@ -743,7 +743,7 @@ func (app *App) syncUpdateRootV3(c *gin.Context) { } uid := c.GetString(userIDKey) - newgeneration, err := app.blobStorer.StoreBlob(uid, "root", bytes.NewBufferString(rootv3.Hash), rootv3.Generation) + newgeneration, err := app.blobStorer.StoreBlob(uid, RootHash, bytes.NewBufferString(rootv3.Hash), rootv3.Generation) if err != nil { log.Error(err) c.AbortWithStatus(http.StatusInternalServerError) @@ -759,24 +759,44 @@ func (app *App) syncUpdateRootV3(c *gin.Context) { } c.JSON(http.StatusOK, messages.SyncRootV3Response{ - Generation: newgeneration, - Hash: rootv3.Hash, - SchemaVersion: SchemaVersion, + Generation: newgeneration, + Hash: rootv3.Hash, }) } const SchemaVersion = 3 -func (app *App) syncGetRoot(c *gin.Context, newAccount func(*gin.Context)) { - uid := c.GetString(userIDKey) +const RmTokenTtlHeader = "Rm-Token-Ttl-Hint" +const RmFileHeader = "rm-filename" + +const RootHash = "root" - reader, generation, _, err := app.blobStorer.LoadBlob(uid, "root") +// crcJSON calculates and ands the crc32c header +// TODO: fix it with a custom render or something +func crcJSON(c *gin.Context, status int, msg any) { + b, err := json.Marshal(msg) + if err != nil { + panic(err) + } + + crc, err := common.CRC32FromReader(bytes.NewBuffer(b)) + if err != nil { + panic(err) + } + common.AddCRCHeader(c, crc) + c.Data(status, "application/json", b) +} + +func (app *App) syncGetRootV3(c *gin.Context) { + uid := c.GetString(userIDKey) + reader, generation, _, _, err := app.blobStorer.LoadBlob(uid, RootHash) if err == fs.ErrorNotFound { log.Warn("No root file found, assuming this is a new account") - newAccount(c) c.JSON(http.StatusNotFound, gin.H{"message": "root not found"}) return - } else if err != nil { + } + + if err != nil { log.Error(err) c.AbortWithStatus(http.StatusInternalServerError) return @@ -790,23 +810,38 @@ func (app *App) syncGetRoot(c *gin.Context, newAccount func(*gin.Context)) { } c.JSON(http.StatusOK, messages.SyncRootV3Response{ - Generation: generation, - Hash: string(roothash), - SchemaVersion: SchemaVersion, - }) -} - -func (app *App) syncGetRootV3(c *gin.Context) { - app.syncGetRoot(c, func(c *gin.Context) { - c.JSON(http.StatusNotFound, gin.H{"message": "root not found"}) + Generation: generation, + Hash: string(roothash), }) } func (app *App) syncGetRootV4(c *gin.Context) { - app.syncGetRoot(c, func(c *gin.Context) { - c.JSON(http.StatusOK, messages.SyncRootV3Response{ - Generation: 0, + uid := c.GetString(userIDKey) + reader, generation, _, _, err := app.blobStorer.LoadBlob(uid, RootHash) + if err == fs.ErrorNotFound { + log.Warn("No root file found, assuming this is a new account") + crcJSON(c, http.StatusOK, messages.SyncRootV4Response{ + SchemaVersion: SchemaVersion, }) + return + } + + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + roothash, err := io.ReadAll(reader) + if err != nil { + log.Error(err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + crcJSON(c, http.StatusOK, messages.SyncRootV4Response{ + Generation: generation, + Hash: string(roothash), + SchemaVersion: SchemaVersion, }) } @@ -843,13 +878,14 @@ func (app *App) blobStorageRead(c *gin.Context) { uid := c.GetString(userIDKey) blobID := common.ParamS(fileKey, c) - reader, _, size, err := app.blobStorer.LoadBlob(uid, blobID) + reader, _, size, crc32c, err := app.blobStorer.LoadBlob(uid, blobID) if err != nil { log.Error(err) c.AbortWithStatus(http.StatusInternalServerError) return } defer reader.Close() + common.AddCRCHeader(c, crc32c) c.DataFromReader(http.StatusOK, size, "application/octet-stream", reader, nil) } @@ -858,6 +894,10 @@ func (app *App) blobStorageWrite(c *gin.Context) { uid := c.GetString(userIDKey) blobID := common.ParamS(fileKey, c) + fileName := c.GetHeader(RmFileHeader) + hash := c.GetHeader(common.CRC32CHashHeader) + log.Debugf("TODO: check/save etc. write file '%s', hash '%s'", fileName, hash) + newgeneration, err := app.blobStorer.StoreBlob(uid, blobID, c.Request.Body, 0) if err != nil { log.Error(err) @@ -865,7 +905,8 @@ func (app *App) blobStorageWrite(c *gin.Context) { return } - c.JSON(http.StatusOK, messages.SyncRootV3Response{ + //not checked by the client yet, but who knows + crcJSON(c, http.StatusOK, messages.SyncRootV4Response{ Generation: newgeneration, Hash: string(blobID), SchemaVersion: SchemaVersion, @@ -1040,6 +1081,27 @@ func (app *App) connectWebSocket(c *gin.Context) { go app.hub.ConnectWs(uid, deviceID, connection) } +// syncReports reports sync errors back +func (app *App) syncReports(c *gin.Context) { + body, err := io.ReadAll(c.Request.Body) + + if err != nil { + log.Warn("cant parse sync report, ignored") + c.Status(http.StatusOK) + return + } + log.Infof("got sync report: %s", string(body)) + c.Status(http.StatusOK) +} + +func (app *App) nullReport(c *gin.Context) { + _, err := io.ReadAll(c.Request.Body) + if err != nil { + log.Warn("could not read report data") + } + c.Status(http.StatusOK) +} + // / remove remarkable ads func stripAds(msg string) string { br := "
--
" diff --git a/internal/app/routes.go b/internal/app/routes.go index 703a56d8..4601251d 100644 --- a/internal/app/routes.go +++ b/internal/app/routes.go @@ -41,6 +41,7 @@ func (app *App) registerRoutes(router *gin.Engine) { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"err": err.Error()}) return } + log.Infof("endpoint %s", endpoint) c.JSON(http.StatusOK, gin.H{ "Status": "OK", "Host": endpoint, @@ -80,27 +81,11 @@ func (app *App) registerRoutes(router *gin.Engine) { c.Status(http.StatusOK) }) + router.POST("/analytics/v2/events", app.nullReport) //some telemetry stuff from ping. - router.POST("/v1/reports", func(c *gin.Context) { - _, err := io.ReadAll(c.Request.Body) - - if err != nil { - log.Warn("cant parse telemetry, ignored") - c.Status(http.StatusOK) - return - } - c.Status(http.StatusOK) - }) - router.POST("/v2/reports", func(c *gin.Context) { - _, err := io.ReadAll(c.Request.Body) - - if err != nil { - log.Warn("cant parse telemetry, ignored") - c.Status(http.StatusOK) - return - } - c.Status(http.StatusOK) - }) + router.POST("/v1/reports", app.nullReport) + router.POST("/v2/reports", app.nullReport) + router.POST("/report/v1", app.nullReport) //routes needing api authentitcation authRoutes := router.Group("/") @@ -159,5 +144,8 @@ func (app *App) registerRoutes(router *gin.Engine) { authRoutes.GET("/sync/v3/missing", app.checkMissingBlob) authRoutes.GET("/sync/v4/root", app.syncGetRootV4) + + // reports + authRoutes.POST("/sync/reports/v1", app.syncReports) } } diff --git a/internal/common/common.go b/internal/common/common.go index 2e1b1251..3be999ad 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -1,7 +1,11 @@ package common import ( + "encoding/base64" + "encoding/binary" "errors" + "hash/crc32" + "io" "regexp" "strings" @@ -62,3 +66,32 @@ func ParamS(param string, c *gin.Context) string { p := c.Param(param) return Sanitize(p) } + +var table = crc32.MakeTable(crc32.Castagnoli) + +func CRC32FromReader(reader io.Reader) (string, error) { + // Create a table for CRC32C (Castagnoli polynomial) + // Create a CRC32C hasher + crc32c := crc32.New(table) + + // Copy the reader data into the hasher + if _, err := io.Copy(crc32c, reader); err != nil { + return "", err + } + + // Compute the CRC32C checksum + checksum := crc32c.Sum32() + + // Convert the checksum to a byte array + crcBytes := make([]byte, 4) + binary.BigEndian.PutUint32(crcBytes, checksum) + encodedChecksum := base64.StdEncoding.EncodeToString(crcBytes) + + return encodedChecksum, nil +} + +const CRC32CHashHeader = "x-goog-hash" + +func AddCRCHeader(c *gin.Context, crc string) { + c.Header(CRC32CHashHeader, "crc32c="+crc) +} diff --git a/internal/messages/messages.go b/internal/messages/messages.go index efab3192..4d4ac03c 100644 --- a/internal/messages/messages.go +++ b/internal/messages/messages.go @@ -145,6 +145,12 @@ type SyncRootV3Request struct { // SyncRootV3Response type SyncRootV3Response struct { + Generation int64 `json:"generation"` + Hash string `json:"hash,omitempty"` +} + +// SyncRootV4Response +type SyncRootV4Response struct { Generation int64 `json:"generation"` Hash string `json:"hash,omitempty"` SchemaVersion int64 `json:"schemaVersion"` diff --git a/internal/storage/fs/app.go b/internal/storage/fs/app.go index bb717315..f0e4cd38 100644 --- a/internal/storage/fs/app.go +++ b/internal/storage/fs/app.go @@ -151,7 +151,7 @@ func (app *App) downloadBlob(c *gin.Context) { log.Info("Requestng blob: ", blobID) - reader, generation, size, err := app.fs.LoadBlob(uid, blobID) + reader, generation, size, crc32c, err := app.fs.LoadBlob(uid, blobID) if err != nil { if err == ErrorNotFound { c.AbortWithStatus(http.StatusNotFound) @@ -163,6 +163,8 @@ func (app *App) downloadBlob(c *gin.Context) { } defer reader.Close() + common.AddCRCHeader(c, crc32c) + if blobID == rootBlob { log.Debug("Sending gen for root: ", generation) c.Header(generationHeader, strconv.FormatInt(generation, 10)) diff --git a/internal/storage/fs/blobstore.go b/internal/storage/fs/blobstore.go index 6f714b39..b05fceca 100644 --- a/internal/storage/fs/blobstore.go +++ b/internal/storage/fs/blobstore.go @@ -454,7 +454,7 @@ func (fs *FileSystemStorage) GetBlobURL(uid, blobid string, write bool) (docurl } // LoadBlob Opens a blob by id -func (fs *FileSystemStorage) LoadBlob(uid, blobid string) (reader io.ReadCloser, gen int64, size int64, err error) { +func (fs *FileSystemStorage) LoadBlob(uid, blobid string) (reader io.ReadCloser, gen int64, size int64, crc32 string, err error) { generation := int64(0) blobPath := path.Join(fs.getUserBlobPath(uid), common.Sanitize(blobid)) log.Debugln("Fullpath:", blobPath) @@ -464,7 +464,7 @@ func (fs *FileSystemStorage) LoadBlob(uid, blobid string) (reader io.ReadCloser, err := lock.LockWithTimeout(time.Duration(time.Second * 5)) if err != nil { log.Error("cannot obtain lock") - return nil, 0, 0, err + return nil, 0, 0, "", err } defer lock.Unlock() @@ -476,11 +476,23 @@ func (fs *FileSystemStorage) LoadBlob(uid, blobid string) (reader io.ReadCloser, fi, err := os.Stat(blobPath) if err != nil || fi.IsDir() { - return nil, generation, 0, ErrorNotFound + return nil, generation, 0, "", ErrorNotFound } - reader, err = os.Open(blobPath) - return reader, generation, fi.Size(), err + osFile, err := os.Open(blobPath) + //TODO: cache the crc32 + crc32, err = common.CRC32FromReader(osFile) + if err != nil { + log.Errorf("cannot get crc32 hash %v", err) + return + } + _, err = osFile.Seek(0, 0) + if err != nil { + log.Errorf("cannot rewind file %v", err) + return + } + reader = osFile + return reader, generation, fi.Size(), crc32, err } // StoreBlob stores a document diff --git a/internal/storage/fs/localblobstorage.go b/internal/storage/fs/localblobstorage.go index ae072869..02fcc46d 100644 --- a/internal/storage/fs/localblobstorage.go +++ b/internal/storage/fs/localblobstorage.go @@ -15,7 +15,7 @@ type LocalBlobStorage struct { // GetRootIndex the hash of the root index func (p *LocalBlobStorage) GetRootIndex() (hash string, gen int64, err error) { - r, gen, _, err := p.fs.LoadBlob(p.uid, rootBlob) + r, gen, _, _, err := p.fs.LoadBlob(p.uid, rootBlob) if err == ErrorNotFound { log.Info("root not found") return "", gen, nil @@ -40,7 +40,7 @@ func (p *LocalBlobStorage) WriteRootIndex(generation int64, roothash string) (in // GetReader reader for a given hash func (p *LocalBlobStorage) GetReader(hash string) (io.ReadCloser, error) { - r, _, _, err := p.fs.LoadBlob(p.uid, hash) + r, _, _, _, err := p.fs.LoadBlob(p.uid, hash) return r, err } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 2ee9d778..21f29dd7 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -34,7 +34,7 @@ type BlobStorage interface { GetBlobURL(uid, docid string, write bool) (string, time.Time, error) StoreBlob(uid, blobID string, s io.Reader, matchGeneration int64) (int64, error) - LoadBlob(uid, blobID string) (reader io.ReadCloser, gen int64, size int64, err error) + LoadBlob(uid, blobID string) (reader io.ReadCloser, gen int64, size int64, crc32c string, err error) CreateBlobDocument(uid, name, parent string, stream io.Reader) (doc *Document, err error) }