Implemented developer database downloads

arn/Analytics.go

@@ -4,7 +4,7 @@ import "github.com/aerogo/nano"
 
 // Analytics stores user-related statistics.
 type Analytics struct {
-	UserID     string              `json:"userId"`
+	UserID     string              `json:"userId" primary:"true"`
 	General    GeneralAnalytics    `json:"general"`
 	Screen     ScreenAnalytics     `json:"screen"`
 	System     SystemAnalytics     `json:"system"`

arn/Anime.go

@@ -67,7 +67,7 @@ type AnimeID = string
 
 // Anime represents an anime.
 type Anime struct {
-	ID            AnimeID          `json:"id"`
+	ID            AnimeID          `json:"id" primary:"true"`
 	Type          string           `json:"type" editable:"true" datalist:"anime-types"`
 	Title         *MediaTitle      `json:"title" editable:"true"`
 	Summary       string           `json:"summary" editable:"true" type:"textarea"`

arn/AnimeCharacters.go

@@ -10,7 +10,7 @@ import (
 
 // AnimeCharacters is a list of characters for an anime.
 type AnimeCharacters struct {
-	AnimeID AnimeID           `json:"animeId" mainID:"true"`
+	AnimeID AnimeID           `json:"animeId" primary:"true"`
 	Items   []*AnimeCharacter `json:"items" editable:"true"`
 
 	sync.Mutex

arn/AnimeList.go

@@ -11,7 +11,7 @@ import (
 
 // AnimeList is a list of anime list items.
 type AnimeList struct {
-	UserID UserID           `json:"userId"`
+	UserID UserID           `json:"userId" primary:"true"`
 	Items  []*AnimeListItem `json:"items"`
 
 	sync.Mutex

arn/AnimeRelations.go

@@ -9,7 +9,7 @@ import (
 
 // AnimeRelations is a list of relations for an anime.
 type AnimeRelations struct {
-	AnimeID AnimeID          `json:"animeId" mainID:"true"`
+	AnimeID AnimeID          `json:"animeId" primary:"true"`
 	Items   []*AnimeRelation `json:"items" editable:"true"`
 
 	sync.Mutex

arn/ClientErrorReport.go

@@ -4,13 +4,13 @@ import "github.com/aerogo/nano"
 
 // ClientErrorReport saves JavaScript errors that happen in web clients like browsers.
 type ClientErrorReport struct {
-	ID           string `json:"id"`
 	Message      string `json:"message"`
 	Stack        string `json:"stack"`
 	FileName     string `json:"fileName"`
 	LineNumber   int    `json:"lineNumber"`
 	ColumnNumber int    `json:"columnNumber"`
 
+	hasID
 	hasCreator
 }
 

arn/DraftIndex.go

@@ -7,7 +7,7 @@ import (
 
 // DraftIndex has references to unpublished drafts a user created.
 type DraftIndex struct {
-	UserID       string  `json:"userId"`
+	UserID       string  `json:"userId" primary:"true"`
 	GroupID      string  `json:"groupId"`
 	SoundTrackID string  `json:"soundTrackId"`
 	CompanyID    string  `json:"companyId"`

arn/EditLogEntry.go

@@ -9,7 +9,7 @@ import (
 
 // EditLogEntry is an entry in the editor log.
 type EditLogEntry struct {
-	ID         string `json:"id"`
+	ID         string `json:"id" primary:"true"`
 	UserID     string `json:"userId"`
 	Action     string `json:"action"`
 	ObjectType string `json:"objectType"` // The typename of what was edited

arn/EmailToUser.go

@@ -2,6 +2,6 @@ package arn
 
 // EmailToUser stores the user ID for an email address.
 type EmailToUser struct {
-	Email  string `json:"email"`
+	Email  string `json:"email" primary:"true"`
 	UserID UserID `json:"userId"`
 }

arn/Episode.go

@@ -9,7 +9,7 @@ import (
 
 // Episode represents a single episode for an anime.
 type Episode struct {
-	ID         string            `json:"id"`
+	ID         string            `json:"id" primary:"true"`
 	AnimeID    AnimeID           `json:"animeId"`
 	Number     int               `json:"number" editable:"true"`
 	Title      EpisodeTitle      `json:"title" editable:"true"`

arn/GoogleToUser.go

@@ -2,6 +2,6 @@ package arn
 
 // GoogleToUser stores the user ID by Google user ID.
 type GoogleToUser struct {
-	ID     string `json:"id"`
+	ID     string `json:"id" primary:"true"`
 	UserID UserID `json:"userId"`
 }

arn/HasID.go

@@ -2,7 +2,7 @@ package arn
 
 // hasID includes an object ID.
 type hasID struct {
-	ID string `json:"id"`
+	ID string `json:"id" primary:"true"`
 }
 
 // GetID returns the ID.

arn/Identifiable.go

@@ -0,0 +1,6 @@
+package arn
+
+// Identifiable applies to any type that has an ID and exposes it via GetID.
+type Identifiable interface {
+	GetID() string
+}

arn/IgnoreAnimeDifference.go

@@ -11,10 +11,10 @@ const IgnoreAnimeDifferenceEditorScore = 2
 
 // IgnoreAnimeDifference saves which differences between anime databases can be ignored.
 type IgnoreAnimeDifference struct {
-	// The ID is built like this: arn:323|mal:356|JapaneseTitle
-	ID        string `json:"id"`
 	ValueHash uint64 `json:"valueHash"`
 
+	// The ID is built like this: arn:323|mal:356|JapaneseTitle
+	hasID
 	hasCreator
 }
 

arn/Inventory.go

@@ -9,7 +9,7 @@ const DefaultInventorySlotCount = 24
 
 // Inventory has inventory slots that store shop item IDs and their quantity.
 type Inventory struct {
-	UserID UserID           `json:"userId"`
+	UserID UserID           `json:"userId" primary:"true"`
 	Slots  []*InventorySlot `json:"slots"`
 }
 

arn/Loggable.go

@@ -9,7 +9,7 @@ import (
 
 // Loggable applies to any type that has a TypeName function.
 type Loggable interface {
-	GetID() string
+	Identifiable
 	TypeName() string
 	Self() Loggable
 }

arn/NickToUser.go

@@ -2,6 +2,6 @@ package arn
 
 // NickToUser stores the user ID by nickname.
 type NickToUser struct {
-	Nick   string `json:"nick"`
+	Nick   string `json:"nick" primary:"true"`
 	UserID UserID `json:"userId"`
 }

arn/Notification.go

@@ -9,10 +9,11 @@ import (
 
 // Notification represents a user-associated notification.
 type Notification struct {
-	ID      string `json:"id"`
 	UserID  string `json:"userId"`
 	Created string `json:"created"`
 	Seen    string `json:"seen"`
+
+	hasID
 	PushNotification
 }
 
@@ -36,7 +37,9 @@ func (notification *Notification) String() string {
 // NewNotification creates a new notification.
 func NewNotification(userID UserID, pushNotification *PushNotification) *Notification {
 	return &Notification{
-		ID:               GenerateID("Notification"),
+		hasID: hasID{
+			ID: GenerateID("Notification"),
+		},
 		UserID:           userID,
 		Created:          DateTimeUTC(),
 		Seen:             "",

arn/PayPalPayment.go

@@ -8,13 +8,29 @@ import (
 
 // PayPalPayment is an approved and exeucted PayPal payment.
 type PayPalPayment struct {
-	ID       string `json:"id"`
 	UserID   string `json:"userId"`
 	PayerID  string `json:"payerId"`
 	Amount   string `json:"amount"`
 	Currency string `json:"currency"`
 	Method   string `json:"method"`
 	Created  string `json:"created"`
+
+	hasID
+}
+
+// NewPayPalPayment creates a new PayPalPayment object with the paypal provided ID.
+func NewPayPalPayment(paymentID, payerID, userID, method, amount, currency string) *PayPalPayment {
+	return &PayPalPayment{
+		hasID: hasID{
+			ID: paymentID,
+		},
+		PayerID:  payerID,
+		UserID:   userID,
+		Method:   method,
+		Amount:   amount,
+		Currency: currency,
+		Created:  DateTimeUTC(),
+	}
 }
 
 // Gems returns the total amount of gems.

arn/Purchase.go

@@ -4,13 +4,14 @@ import "github.com/aerogo/nano"
 
 // Purchase represents an item purchase by a user.
 type Purchase struct {
-	ID       string `json:"id"`
 	UserID   string `json:"userId"`
 	ItemID   string `json:"itemId"`
 	Quantity int    `json:"quantity"`
 	Price    int    `json:"price"`
 	Currency string `json:"currency"`
 	Date     string `json:"date"`
+
+	hasID
 }
 
 // Item returns the item the user bought.
@@ -28,7 +29,9 @@ func (purchase *Purchase) User() *User {
 // NewPurchase creates a new Purchase object with a generated ID.
 func NewPurchase(userID UserID, itemID string, quantity int, price int, currency string) *Purchase {
 	return &Purchase{
-		ID:       GenerateID("Purchase"),
+		hasID: hasID{
+			ID: GenerateID("Purchase"),
+		},
 		UserID:   userID,
 		ItemID:   itemID,
 		Quantity: quantity,

arn/PushSubscriptions.go

@@ -4,7 +4,7 @@ import "errors"
 
 // PushSubscriptions is a list of push subscriptions made by a user.
 type PushSubscriptions struct {
-	UserID UserID              `json:"userId"`
+	UserID UserID              `json:"userId" primary:"true"`
 	Items  []*PushSubscription `json:"items"`
 }
 

arn/Settings.go

@@ -29,7 +29,7 @@ const (
 
 // Settings represents user settings.
 type Settings struct {
-	UserID        string               `json:"userId"`
+	UserID        string               `json:"userId" primary:"true"`
 	SortBy        string               `json:"sortBy" editable:"true"`
 	TitleLanguage string               `json:"titleLanguage" editable:"true"`
 	Providers     ServiceProviders     `json:"providers"`
@@ -162,6 +162,11 @@ func GetSettings(userID UserID) (*Settings, error) {
 	return obj.(*Settings), nil
 }
 
+// GetID returns the ID.
+func (settings *Settings) GetID() string {
+	return settings.UserID
+}
+
 // User returns the user object for the settings.
 func (settings *Settings) User() *User {
 	user, _ := GetUser(settings.UserID)

arn/SettingsAPI.go

@@ -10,6 +10,7 @@ import (
 
 // Force interface implementations
 var (
+	_ Identifiable = (*Settings)(nil)
 	_ api.Editable = (*Settings)(nil)
 	_ api.Filter   = (*Settings)(nil)
 )

arn/ShopItem.go

@@ -21,7 +21,6 @@ const (
 
 // ShopItem is a purchasable item in the shop.
 type ShopItem struct {
-	ID          string `json:"id"`
 	Name        string `json:"name"`
 	Description string `json:"description"`
 	Price       uint   `json:"price"`
@@ -29,6 +28,8 @@ type ShopItem struct {
 	Rarity      string `json:"rarity"`
 	Order       int    `json:"order"`
 	Consumable  bool   `json:"consumable"`
+
+	hasID
 }
 
 // GetShopItem ...

arn/User.go

@@ -37,7 +37,7 @@ type UserID = string
 
 // User is a registered person.
 type User struct {
-	ID           UserID       `json:"id"`
+	ID           UserID       `json:"id" primary:"true"`
 	Nick         string       `json:"nick" editable:"true"`
 	FirstName    string       `json:"firstName" private:"true"`
 	LastName     string       `json:"lastName" private:"true"`

arn/UserFollows.go

@@ -8,7 +8,7 @@ import (
 
 // UserFollows is a list including IDs to users you follow.
 type UserFollows struct {
-	UserID UserID   `json:"userId"`
+	UserID UserID   `json:"userId" primary:"true"`
 	Items  []string `json:"items"`
 }
 

arn/UserNotifications.go

@@ -8,7 +8,7 @@ import (
 
 // UserNotifications is a list including IDs to your notifications.
 type UserNotifications struct {
-	UserID UserID   `json:"userId"`
+	UserID UserID   `json:"userId" primary:"true"`
 	Items  []string `json:"items"`
 }
 

go.mod

@@ -49,6 +49,7 @@ require (
 	github.com/mailgun/mailgun-go/v3 v3.6.0
 	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/minio/minio-go/v6 v6.0.34
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/mssola/user_agent v0.5.0
 	github.com/pariz/gountries v0.0.0-20171019111738-adb00f6513a3
 	github.com/shirou/gopsutil v2.18.12+incompatible

pages/database/download.go

@@ -1,10 +1,90 @@
 package database
 
 import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
 	"github.com/aerogo/aero"
+	"github.com/aerogo/api"
+	"github.com/akyoto/stringutils/unsafe"
+	"github.com/animenotifier/notify.moe/arn"
+	"github.com/mohae/deepcopy"
 )
 
+// privateTypes are types that are not available for download.
+var privateTypes = []string{
+	"EditLogEntry",
+	"EmailToUser",
+	"PayPalPayment",
+	"Purchase",
+	"Session",
+}
+
 // Download downloads a snapshot of a database collection.
 func Download(ctx aero.Context) error {
-	return nil
+	typ := ctx.Get("type")
+
+	if !arn.DB.HasType(typ) {
+		return ctx.Error(http.StatusNotFound, "Type doesn't exist")
+	}
+
+	if arn.Contains(privateTypes, typ) {
+		return ctx.Error(http.StatusUnauthorized, "Type is private and can not be downloaded")
+	}
+
+	// Send headers necessary for file downloads
+	ctx.Response().SetHeader("Content-Disposition", fmt.Sprintf(`attachment; filename="%s.dat"`, typ))
+
+	// Stream data
+	reader, writer := io.Pipe()
+	encoder := json.NewEncoder(writer)
+
+	go func() {
+		for object := range arn.DB.All(typ) {
+			idObject, hasID := object.(arn.Identifiable)
+
+			if !hasID {
+				continue
+			}
+
+			// Filter out private data
+			filter, isFilter := object.(api.Filter)
+
+			if isFilter && filter.ShouldFilter(ctx) {
+				object = deepcopy.Copy(object)
+				filter = object.(api.Filter)
+				filter.Filter()
+			}
+
+			// Write ID
+			_, err := writer.Write(unsafe.StringToBytes(idObject.GetID()))
+
+			if err != nil {
+				_ = writer.CloseWithError(err)
+				return
+			}
+
+			// Write newline
+			_, err = writer.Write([]byte("\n"))
+
+			if err != nil {
+				_ = writer.CloseWithError(err)
+				return
+			}
+
+			// Write JSON (newline included)
+			err = encoder.Encode(object)
+
+			if err != nil {
+				_ = writer.CloseWithError(err)
+				return
+			}
+		}
+
+		writer.Close()
+	}()
+
+	return ctx.Reader(reader)
 }

pages/database/types.go

@@ -13,6 +13,10 @@ func Types(ctx aero.Context) error {
 	types := make([]string, 0, len(typeMap))
 
 	for typeName := range typeMap {
+		if arn.Contains(privateTypes, typeName) {
+			continue
+		}
+
 		types = append(types, typeName)
 	}
 

pages/index/apiroutes/apiroutes.go

@@ -54,7 +54,7 @@ func Register(app *aero.Application) {
 
 	// Types
 	app.Get("/api/types", database.Types)
-	app.Get("/api/types/:type/all", database.Download)
+	app.Get("/api/types/:type/download", database.Download)
 
 	// SoundTrack
 	app.Post("/api/soundtrack/:id/download", soundtrack.Download)

pages/paypal/success.go

@@ -63,19 +63,8 @@ func Success(ctx aero.Context) error {
 	}
 
 	stringutils.PrettyPrint(sdkPayment)
-
 	transaction := sdkPayment.Transactions[0]
-
-	payment := &arn.PayPalPayment{
-		ID:       paymentID,
-		PayerID:  payerID,
-		UserID:   user.ID,
-		Method:   sdkPayment.Payer.PaymentMethod,
-		Amount:   transaction.Amount.Total,
-		Currency: transaction.Amount.Currency,
-		Created:  arn.DateTimeUTC(),
-	}
-
+	payment := arn.NewPayPalPayment(paymentID, payerID, user.ID, sdkPayment.Payer.PaymentMethod, transaction.Amount.Total, transaction.Amount.Currency)
 	payment.Save()
 
 	// Increase user's balance

utils/editform/editform.go

@@ -18,7 +18,7 @@ import (
 func Render(obj interface{}, title string, user *arn.User) string {
 	t := reflect.TypeOf(obj).Elem()
 	v := reflect.ValueOf(obj).Elem()
-	id := findMainID(t, v)
+	id := findPrimaryID(t, v)
 	lowerCaseTypeName := strings.ToLower(t.Name())
 	endpoint := `/api/` + lowerCaseTypeName + `/` + id.String()
 
@@ -319,8 +319,8 @@ func renderSliceField(b *strings.Builder, field reflect.StructField, idPrefix st
 	b.WriteString(`</div>`)
 }
 
-// findMainID finds the main ID of the object.
-func findMainID(t reflect.Type, v reflect.Value) reflect.Value {
+// findPrimaryID finds the primary ID of the object.
+func findPrimaryID(t reflect.Type, v reflect.Value) reflect.Value {
 	idField := v.FieldByName("ID")
 
 	if idField.IsValid() {
@@ -330,7 +330,7 @@ func findMainID(t reflect.Type, v reflect.Value) reflect.Value {
 	for i := 0; i < t.NumField(); i++ {
 		field := t.Field(i)
 
-		if field.Tag.Get("mainID") == "true" {
+		if field.Tag.Get("primary") == "true" {
 			return reflect.Indirect(v.Field(i))
 		}
 	}