From 4e6aa5eef65fbc2fcd2a22b7633a578a11d8f990 Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Sat, 7 Sep 2019 19:56:13 +0900 Subject: [PATCH] Implemented developer database downloads --- arn/Analytics.go | 2 +- arn/Anime.go | 2 +- arn/AnimeCharacters.go | 2 +- arn/AnimeList.go | 2 +- arn/AnimeRelations.go | 2 +- arn/ClientErrorReport.go | 2 +- arn/DraftIndex.go | 2 +- arn/EditLogEntry.go | 2 +- arn/EmailToUser.go | 2 +- arn/Episode.go | 2 +- arn/GoogleToUser.go | 2 +- arn/HasID.go | 2 +- arn/Identifiable.go | 6 +++ arn/IgnoreAnimeDifference.go | 4 +- arn/Inventory.go | 2 +- arn/Loggable.go | 2 +- arn/NickToUser.go | 2 +- arn/Notification.go | 7 ++- arn/PayPalPayment.go | 18 ++++++- arn/Purchase.go | 7 ++- arn/PushSubscriptions.go | 2 +- arn/Settings.go | 7 ++- arn/SettingsAPI.go | 1 + arn/ShopItem.go | 3 +- arn/User.go | 2 +- arn/UserFollows.go | 2 +- arn/UserNotifications.go | 2 +- go.mod | 1 + pages/database/download.go | 82 +++++++++++++++++++++++++++++- pages/database/types.go | 4 ++ pages/index/apiroutes/apiroutes.go | 2 +- pages/paypal/success.go | 13 +---- utils/editform/editform.go | 8 +-- 33 files changed, 155 insertions(+), 46 deletions(-) create mode 100644 arn/Identifiable.go diff --git a/arn/Analytics.go b/arn/Analytics.go index 6dd120cb..0d1cb3a3 100644 --- a/arn/Analytics.go +++ b/arn/Analytics.go @@ -4,7 +4,7 @@ import "github.com/aerogo/nano" // Analytics stores user-related statistics. type Analytics struct { - UserID string `json:"userId"` + UserID string `json:"userId" primary:"true"` General GeneralAnalytics `json:"general"` Screen ScreenAnalytics `json:"screen"` System SystemAnalytics `json:"system"` diff --git a/arn/Anime.go b/arn/Anime.go index a06478c4..2dcd67e6 100644 --- a/arn/Anime.go +++ b/arn/Anime.go @@ -67,7 +67,7 @@ type AnimeID = string // Anime represents an anime. type Anime struct { - ID AnimeID `json:"id"` + ID AnimeID `json:"id" primary:"true"` Type string `json:"type" editable:"true" datalist:"anime-types"` Title *MediaTitle `json:"title" editable:"true"` Summary string `json:"summary" editable:"true" type:"textarea"` diff --git a/arn/AnimeCharacters.go b/arn/AnimeCharacters.go index c6a79698..ee25ff50 100644 --- a/arn/AnimeCharacters.go +++ b/arn/AnimeCharacters.go @@ -10,7 +10,7 @@ import ( // AnimeCharacters is a list of characters for an anime. type AnimeCharacters struct { - AnimeID AnimeID `json:"animeId" mainID:"true"` + AnimeID AnimeID `json:"animeId" primary:"true"` Items []*AnimeCharacter `json:"items" editable:"true"` sync.Mutex diff --git a/arn/AnimeList.go b/arn/AnimeList.go index 0979f366..2dbb4db7 100644 --- a/arn/AnimeList.go +++ b/arn/AnimeList.go @@ -11,7 +11,7 @@ import ( // AnimeList is a list of anime list items. type AnimeList struct { - UserID UserID `json:"userId"` + UserID UserID `json:"userId" primary:"true"` Items []*AnimeListItem `json:"items"` sync.Mutex diff --git a/arn/AnimeRelations.go b/arn/AnimeRelations.go index 35f8a607..1a099d0f 100644 --- a/arn/AnimeRelations.go +++ b/arn/AnimeRelations.go @@ -9,7 +9,7 @@ import ( // AnimeRelations is a list of relations for an anime. type AnimeRelations struct { - AnimeID AnimeID `json:"animeId" mainID:"true"` + AnimeID AnimeID `json:"animeId" primary:"true"` Items []*AnimeRelation `json:"items" editable:"true"` sync.Mutex diff --git a/arn/ClientErrorReport.go b/arn/ClientErrorReport.go index df3dd017..4c9d6d8a 100644 --- a/arn/ClientErrorReport.go +++ b/arn/ClientErrorReport.go @@ -4,13 +4,13 @@ import "github.com/aerogo/nano" // ClientErrorReport saves JavaScript errors that happen in web clients like browsers. type ClientErrorReport struct { - ID string `json:"id"` Message string `json:"message"` Stack string `json:"stack"` FileName string `json:"fileName"` LineNumber int `json:"lineNumber"` ColumnNumber int `json:"columnNumber"` + hasID hasCreator } diff --git a/arn/DraftIndex.go b/arn/DraftIndex.go index 22324c65..81743f45 100644 --- a/arn/DraftIndex.go +++ b/arn/DraftIndex.go @@ -7,7 +7,7 @@ import ( // DraftIndex has references to unpublished drafts a user created. type DraftIndex struct { - UserID string `json:"userId"` + UserID string `json:"userId" primary:"true"` GroupID string `json:"groupId"` SoundTrackID string `json:"soundTrackId"` CompanyID string `json:"companyId"` diff --git a/arn/EditLogEntry.go b/arn/EditLogEntry.go index 787da48d..04ba7109 100644 --- a/arn/EditLogEntry.go +++ b/arn/EditLogEntry.go @@ -9,7 +9,7 @@ import ( // EditLogEntry is an entry in the editor log. type EditLogEntry struct { - ID string `json:"id"` + ID string `json:"id" primary:"true"` UserID string `json:"userId"` Action string `json:"action"` ObjectType string `json:"objectType"` // The typename of what was edited diff --git a/arn/EmailToUser.go b/arn/EmailToUser.go index 15a32494..7229bd0a 100644 --- a/arn/EmailToUser.go +++ b/arn/EmailToUser.go @@ -2,6 +2,6 @@ package arn // EmailToUser stores the user ID for an email address. type EmailToUser struct { - Email string `json:"email"` + Email string `json:"email" primary:"true"` UserID UserID `json:"userId"` } diff --git a/arn/Episode.go b/arn/Episode.go index a63575ae..75c5a1c0 100644 --- a/arn/Episode.go +++ b/arn/Episode.go @@ -9,7 +9,7 @@ import ( // Episode represents a single episode for an anime. type Episode struct { - ID string `json:"id"` + ID string `json:"id" primary:"true"` AnimeID AnimeID `json:"animeId"` Number int `json:"number" editable:"true"` Title EpisodeTitle `json:"title" editable:"true"` diff --git a/arn/GoogleToUser.go b/arn/GoogleToUser.go index 2008f8a0..3fbb5e3d 100644 --- a/arn/GoogleToUser.go +++ b/arn/GoogleToUser.go @@ -2,6 +2,6 @@ package arn // GoogleToUser stores the user ID by Google user ID. type GoogleToUser struct { - ID string `json:"id"` + ID string `json:"id" primary:"true"` UserID UserID `json:"userId"` } diff --git a/arn/HasID.go b/arn/HasID.go index 73a05d8a..f69e0603 100644 --- a/arn/HasID.go +++ b/arn/HasID.go @@ -2,7 +2,7 @@ package arn // hasID includes an object ID. type hasID struct { - ID string `json:"id"` + ID string `json:"id" primary:"true"` } // GetID returns the ID. diff --git a/arn/Identifiable.go b/arn/Identifiable.go new file mode 100644 index 00000000..d811f226 --- /dev/null +++ b/arn/Identifiable.go @@ -0,0 +1,6 @@ +package arn + +// Identifiable applies to any type that has an ID and exposes it via GetID. +type Identifiable interface { + GetID() string +} diff --git a/arn/IgnoreAnimeDifference.go b/arn/IgnoreAnimeDifference.go index 1d56a90c..3fea42c3 100644 --- a/arn/IgnoreAnimeDifference.go +++ b/arn/IgnoreAnimeDifference.go @@ -11,10 +11,10 @@ const IgnoreAnimeDifferenceEditorScore = 2 // IgnoreAnimeDifference saves which differences between anime databases can be ignored. type IgnoreAnimeDifference struct { - // The ID is built like this: arn:323|mal:356|JapaneseTitle - ID string `json:"id"` ValueHash uint64 `json:"valueHash"` + // The ID is built like this: arn:323|mal:356|JapaneseTitle + hasID hasCreator } diff --git a/arn/Inventory.go b/arn/Inventory.go index c29cc080..f580ed64 100644 --- a/arn/Inventory.go +++ b/arn/Inventory.go @@ -9,7 +9,7 @@ const DefaultInventorySlotCount = 24 // Inventory has inventory slots that store shop item IDs and their quantity. type Inventory struct { - UserID UserID `json:"userId"` + UserID UserID `json:"userId" primary:"true"` Slots []*InventorySlot `json:"slots"` } diff --git a/arn/Loggable.go b/arn/Loggable.go index 2f2b1a80..f73c8bd7 100644 --- a/arn/Loggable.go +++ b/arn/Loggable.go @@ -9,7 +9,7 @@ import ( // Loggable applies to any type that has a TypeName function. type Loggable interface { - GetID() string + Identifiable TypeName() string Self() Loggable } diff --git a/arn/NickToUser.go b/arn/NickToUser.go index ac201638..d941c79d 100644 --- a/arn/NickToUser.go +++ b/arn/NickToUser.go @@ -2,6 +2,6 @@ package arn // NickToUser stores the user ID by nickname. type NickToUser struct { - Nick string `json:"nick"` + Nick string `json:"nick" primary:"true"` UserID UserID `json:"userId"` } diff --git a/arn/Notification.go b/arn/Notification.go index d7e288f2..333edb72 100644 --- a/arn/Notification.go +++ b/arn/Notification.go @@ -9,10 +9,11 @@ import ( // Notification represents a user-associated notification. type Notification struct { - ID string `json:"id"` UserID string `json:"userId"` Created string `json:"created"` Seen string `json:"seen"` + + hasID PushNotification } @@ -36,7 +37,9 @@ func (notification *Notification) String() string { // NewNotification creates a new notification. func NewNotification(userID UserID, pushNotification *PushNotification) *Notification { return &Notification{ - ID: GenerateID("Notification"), + hasID: hasID{ + ID: GenerateID("Notification"), + }, UserID: userID, Created: DateTimeUTC(), Seen: "", diff --git a/arn/PayPalPayment.go b/arn/PayPalPayment.go index ae5bd444..42930ad4 100644 --- a/arn/PayPalPayment.go +++ b/arn/PayPalPayment.go @@ -8,13 +8,29 @@ import ( // PayPalPayment is an approved and exeucted PayPal payment. type PayPalPayment struct { - ID string `json:"id"` UserID string `json:"userId"` PayerID string `json:"payerId"` Amount string `json:"amount"` Currency string `json:"currency"` Method string `json:"method"` Created string `json:"created"` + + hasID +} + +// NewPayPalPayment creates a new PayPalPayment object with the paypal provided ID. +func NewPayPalPayment(paymentID, payerID, userID, method, amount, currency string) *PayPalPayment { + return &PayPalPayment{ + hasID: hasID{ + ID: paymentID, + }, + PayerID: payerID, + UserID: userID, + Method: method, + Amount: amount, + Currency: currency, + Created: DateTimeUTC(), + } } // Gems returns the total amount of gems. diff --git a/arn/Purchase.go b/arn/Purchase.go index bfb17ac9..0dfe4239 100644 --- a/arn/Purchase.go +++ b/arn/Purchase.go @@ -4,13 +4,14 @@ import "github.com/aerogo/nano" // Purchase represents an item purchase by a user. type Purchase struct { - ID string `json:"id"` UserID string `json:"userId"` ItemID string `json:"itemId"` Quantity int `json:"quantity"` Price int `json:"price"` Currency string `json:"currency"` Date string `json:"date"` + + hasID } // Item returns the item the user bought. @@ -28,7 +29,9 @@ func (purchase *Purchase) User() *User { // NewPurchase creates a new Purchase object with a generated ID. func NewPurchase(userID UserID, itemID string, quantity int, price int, currency string) *Purchase { return &Purchase{ - ID: GenerateID("Purchase"), + hasID: hasID{ + ID: GenerateID("Purchase"), + }, UserID: userID, ItemID: itemID, Quantity: quantity, diff --git a/arn/PushSubscriptions.go b/arn/PushSubscriptions.go index 3dbc9ac6..c97296e8 100644 --- a/arn/PushSubscriptions.go +++ b/arn/PushSubscriptions.go @@ -4,7 +4,7 @@ import "errors" // PushSubscriptions is a list of push subscriptions made by a user. type PushSubscriptions struct { - UserID UserID `json:"userId"` + UserID UserID `json:"userId" primary:"true"` Items []*PushSubscription `json:"items"` } diff --git a/arn/Settings.go b/arn/Settings.go index d8c13ea8..7da8b032 100644 --- a/arn/Settings.go +++ b/arn/Settings.go @@ -29,7 +29,7 @@ const ( // Settings represents user settings. type Settings struct { - UserID string `json:"userId"` + UserID string `json:"userId" primary:"true"` SortBy string `json:"sortBy" editable:"true"` TitleLanguage string `json:"titleLanguage" editable:"true"` Providers ServiceProviders `json:"providers"` @@ -162,6 +162,11 @@ func GetSettings(userID UserID) (*Settings, error) { return obj.(*Settings), nil } +// GetID returns the ID. +func (settings *Settings) GetID() string { + return settings.UserID +} + // User returns the user object for the settings. func (settings *Settings) User() *User { user, _ := GetUser(settings.UserID) diff --git a/arn/SettingsAPI.go b/arn/SettingsAPI.go index 039ebe9f..e8feda41 100644 --- a/arn/SettingsAPI.go +++ b/arn/SettingsAPI.go @@ -10,6 +10,7 @@ import ( // Force interface implementations var ( + _ Identifiable = (*Settings)(nil) _ api.Editable = (*Settings)(nil) _ api.Filter = (*Settings)(nil) ) diff --git a/arn/ShopItem.go b/arn/ShopItem.go index ace98e53..129710f4 100644 --- a/arn/ShopItem.go +++ b/arn/ShopItem.go @@ -21,7 +21,6 @@ const ( // ShopItem is a purchasable item in the shop. type ShopItem struct { - ID string `json:"id"` Name string `json:"name"` Description string `json:"description"` Price uint `json:"price"` @@ -29,6 +28,8 @@ type ShopItem struct { Rarity string `json:"rarity"` Order int `json:"order"` Consumable bool `json:"consumable"` + + hasID } // GetShopItem ... diff --git a/arn/User.go b/arn/User.go index a16c95e6..8a21525e 100644 --- a/arn/User.go +++ b/arn/User.go @@ -37,7 +37,7 @@ type UserID = string // User is a registered person. type User struct { - ID UserID `json:"id"` + ID UserID `json:"id" primary:"true"` Nick string `json:"nick" editable:"true"` FirstName string `json:"firstName" private:"true"` LastName string `json:"lastName" private:"true"` diff --git a/arn/UserFollows.go b/arn/UserFollows.go index 4d0cfdb7..d5a239fa 100644 --- a/arn/UserFollows.go +++ b/arn/UserFollows.go @@ -8,7 +8,7 @@ import ( // UserFollows is a list including IDs to users you follow. type UserFollows struct { - UserID UserID `json:"userId"` + UserID UserID `json:"userId" primary:"true"` Items []string `json:"items"` } diff --git a/arn/UserNotifications.go b/arn/UserNotifications.go index d021438e..1d321d9b 100644 --- a/arn/UserNotifications.go +++ b/arn/UserNotifications.go @@ -8,7 +8,7 @@ import ( // UserNotifications is a list including IDs to your notifications. type UserNotifications struct { - UserID UserID `json:"userId"` + UserID UserID `json:"userId" primary:"true"` Items []string `json:"items"` } diff --git a/go.mod b/go.mod index 7172f4c1..5de70bf7 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/mailgun/mailgun-go/v3 v3.6.0 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect github.com/minio/minio-go/v6 v6.0.34 + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/mssola/user_agent v0.5.0 github.com/pariz/gountries v0.0.0-20171019111738-adb00f6513a3 github.com/shirou/gopsutil v2.18.12+incompatible diff --git a/pages/database/download.go b/pages/database/download.go index 1b2caf4f..06373fdd 100644 --- a/pages/database/download.go +++ b/pages/database/download.go @@ -1,10 +1,90 @@ package database import ( + "encoding/json" + "fmt" + "io" + "net/http" + "github.com/aerogo/aero" + "github.com/aerogo/api" + "github.com/akyoto/stringutils/unsafe" + "github.com/animenotifier/notify.moe/arn" + "github.com/mohae/deepcopy" ) +// privateTypes are types that are not available for download. +var privateTypes = []string{ + "EditLogEntry", + "EmailToUser", + "PayPalPayment", + "Purchase", + "Session", +} + // Download downloads a snapshot of a database collection. func Download(ctx aero.Context) error { - return nil + typ := ctx.Get("type") + + if !arn.DB.HasType(typ) { + return ctx.Error(http.StatusNotFound, "Type doesn't exist") + } + + if arn.Contains(privateTypes, typ) { + return ctx.Error(http.StatusUnauthorized, "Type is private and can not be downloaded") + } + + // Send headers necessary for file downloads + ctx.Response().SetHeader("Content-Disposition", fmt.Sprintf(`attachment; filename="%s.dat"`, typ)) + + // Stream data + reader, writer := io.Pipe() + encoder := json.NewEncoder(writer) + + go func() { + for object := range arn.DB.All(typ) { + idObject, hasID := object.(arn.Identifiable) + + if !hasID { + continue + } + + // Filter out private data + filter, isFilter := object.(api.Filter) + + if isFilter && filter.ShouldFilter(ctx) { + object = deepcopy.Copy(object) + filter = object.(api.Filter) + filter.Filter() + } + + // Write ID + _, err := writer.Write(unsafe.StringToBytes(idObject.GetID())) + + if err != nil { + _ = writer.CloseWithError(err) + return + } + + // Write newline + _, err = writer.Write([]byte("\n")) + + if err != nil { + _ = writer.CloseWithError(err) + return + } + + // Write JSON (newline included) + err = encoder.Encode(object) + + if err != nil { + _ = writer.CloseWithError(err) + return + } + } + + writer.Close() + }() + + return ctx.Reader(reader) } diff --git a/pages/database/types.go b/pages/database/types.go index 6ad20668..0c169273 100644 --- a/pages/database/types.go +++ b/pages/database/types.go @@ -13,6 +13,10 @@ func Types(ctx aero.Context) error { types := make([]string, 0, len(typeMap)) for typeName := range typeMap { + if arn.Contains(privateTypes, typeName) { + continue + } + types = append(types, typeName) } diff --git a/pages/index/apiroutes/apiroutes.go b/pages/index/apiroutes/apiroutes.go index 42f16173..866cd4ba 100644 --- a/pages/index/apiroutes/apiroutes.go +++ b/pages/index/apiroutes/apiroutes.go @@ -54,7 +54,7 @@ func Register(app *aero.Application) { // Types app.Get("/api/types", database.Types) - app.Get("/api/types/:type/all", database.Download) + app.Get("/api/types/:type/download", database.Download) // SoundTrack app.Post("/api/soundtrack/:id/download", soundtrack.Download) diff --git a/pages/paypal/success.go b/pages/paypal/success.go index 9320902f..f2b60fa6 100644 --- a/pages/paypal/success.go +++ b/pages/paypal/success.go @@ -63,19 +63,8 @@ func Success(ctx aero.Context) error { } stringutils.PrettyPrint(sdkPayment) - transaction := sdkPayment.Transactions[0] - - payment := &arn.PayPalPayment{ - ID: paymentID, - PayerID: payerID, - UserID: user.ID, - Method: sdkPayment.Payer.PaymentMethod, - Amount: transaction.Amount.Total, - Currency: transaction.Amount.Currency, - Created: arn.DateTimeUTC(), - } - + payment := arn.NewPayPalPayment(paymentID, payerID, user.ID, sdkPayment.Payer.PaymentMethod, transaction.Amount.Total, transaction.Amount.Currency) payment.Save() // Increase user's balance diff --git a/utils/editform/editform.go b/utils/editform/editform.go index 589325c4..51a91ade 100644 --- a/utils/editform/editform.go +++ b/utils/editform/editform.go @@ -18,7 +18,7 @@ import ( func Render(obj interface{}, title string, user *arn.User) string { t := reflect.TypeOf(obj).Elem() v := reflect.ValueOf(obj).Elem() - id := findMainID(t, v) + id := findPrimaryID(t, v) lowerCaseTypeName := strings.ToLower(t.Name()) endpoint := `/api/` + lowerCaseTypeName + `/` + id.String() @@ -319,8 +319,8 @@ func renderSliceField(b *strings.Builder, field reflect.StructField, idPrefix st b.WriteString(``) } -// findMainID finds the main ID of the object. -func findMainID(t reflect.Type, v reflect.Value) reflect.Value { +// findPrimaryID finds the primary ID of the object. +func findPrimaryID(t reflect.Type, v reflect.Value) reflect.Value { idField := v.FieldByName("ID") if idField.IsValid() { @@ -330,7 +330,7 @@ func findMainID(t reflect.Type, v reflect.Value) reflect.Value { for i := 0; i < t.NumField(); i++ { field := t.Field(i) - if field.Tag.Get("mainID") == "true" { + if field.Tag.Get("primary") == "true" { return reflect.Indirect(v.Field(i)) } }