commit bc3ec64b3c8cade845c498f1ac1dd112f5df2861 Author: ByteDream Date: Thu Dec 16 10:08:34 2021 +0100 Initial release diff --git a/.env b/.env new file mode 100644 index 0000000..b3140ba --- /dev/null +++ b/.env @@ -0,0 +1,5 @@ +ARTICLE_ROOT=../articles/ + +SERVER_PORT=8080 + +DATABASE_FILE=database.sqlite3 diff --git a/README.md b/README.md new file mode 100644 index 0000000..18803d0 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# backend diff --git a/api/api.go b/api/api.go new file mode 100644 index 0000000..00dd8d4 --- /dev/null +++ b/api/api.go @@ -0,0 +1,17 @@ +package api + +import ( + "encoding/json" + "net/http" +) + +type ApiError struct { + Message string `json:"message"` + OriginalError error + Code int +} + +func (ae ApiError) Send(w http.ResponseWriter) error { + w.WriteHeader(ae.Code) + return json.NewEncoder(w).Encode(ae) +} diff --git a/api/recent.go b/api/recent.go new file mode 100644 index 0000000..468d0c3 --- /dev/null +++ b/api/recent.go @@ -0,0 +1,39 @@ +package api + +import ( + "TheAdversary/database" + "TheAdversary/schema" + "encoding/json" + "net/http" + "strconv" +) + +func Recent(w http.ResponseWriter, r *http.Request) { + var err error + var limit int64 + + query := r.URL.Query() + if l := query.Get("limit"); l != "" { + limit, err = strconv.ParseInt(l, 10, 64) + if err != nil { + ApiError{"invalid 'limit' parameter", err, http.StatusUnprocessableEntity}.Send(w) + return + } + } else { + limit = 20 + } + + articles, err := database.GetDB().GetArticles("", database.ArticleQueryOptions{ + Limit: int(limit), + }) + + var articleSummaries []schema.ArticleSummary + for _, article := range articles { + articleSummaries = append(articleSummaries, article.ToArticleSummary()) + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(struct { + Articles []schema.ArticleSummary + }{articleSummaries}) +} diff --git a/api/search.go b/api/search.go new file mode 100644 index 0000000..b4befc9 --- /dev/null +++ b/api/search.go @@ -0,0 +1,61 @@ +package api + +import ( + "TheAdversary/database" + "TheAdversary/schema" + "encoding/json" + "net/http" + "strconv" + "time" +) + +func Search(w http.ResponseWriter, r *http.Request) { + var err error + var q string + var from, to time.Time + var limit int64 + + query := r.URL.Query() + q = query.Get("q") + if f := query.Get("from"); f != "" { + from, err = time.Parse(time.RFC3339, f) + if err != nil { + ApiError{"could not parse 'from' parameter as RFC3339 time", err, http.StatusUnprocessableEntity}.Send(w) + return + } + } + if t := query.Get("to"); t != "" { + to, err = time.Parse(time.RFC3339, t) + if err != nil { + ApiError{"could not parse 'to' parameter as RFC3339 time", err, http.StatusUnprocessableEntity}.Send(w) + return + } + } + if l := query.Get("limit"); l != "" { + limit, err = strconv.ParseInt(l, 10, 64) + if err != nil { + ApiError{"invalid 'limit' parameter", err, http.StatusUnprocessableEntity}.Send(w) + return + } + } else { + limit = 20 + } + + articles, err := database.GetDB().GetArticles(q, database.ArticleQueryOptions{ + Title: true, + Summary: true, + From: from.Unix(), + To: to.Unix(), + Limit: int(limit), + }) + + var articleSummaries []schema.ArticleSummary + for _, article := range articles { + articleSummaries = append(articleSummaries, article.ToArticleSummary()) + } + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(struct { + Articles []schema.ArticleSummary + }{articleSummaries}) +} diff --git a/article/article.go b/article/article.go new file mode 100644 index 0000000..8511871 --- /dev/null +++ b/article/article.go @@ -0,0 +1,66 @@ +package article + +import ( + "TheAdversary/config" + "TheAdversary/parse" + "TheAdversary/schema" + "github.com/gomarkdown/markdown/ast" + "path/filepath" + "strings" + "time" +) + +func LoadArticle(path string) (*schema.Article, error) { + node, err := parse.Parse(path) + if err != nil { + return nil, err + } + article := &schema.Article{ + Name: ArticleName(path), + Added: time.Now().Unix(), + } + children := node.GetChildren() + to := 3 + if len(children) < to { + to = len(children) + } + for _, child := range children[0:to] { + switch child.(type) { + case *ast.Heading: + if article.Title != "" { + article.Summary = extractText(child.(*ast.Heading).Container) + } else { + article.Title = extractText(child.(*ast.Heading).Container) + } + case *ast.Paragraph: + if article.Summary == "" { + article.Summary = extractText(child.(*ast.Paragraph).Container) + } + case *ast.BlockQuote: + if article.Summary == "" { + article.Summary = extractText(child.(*ast.BlockQuote).Container) + } + case *ast.Image: + article.Image = string(child.(*ast.Image).Destination) + } + } + + if article.Title == "" { + article.Title = strings.ReplaceAll(strings.ReplaceAll(ArticleName(path), "-", " "), "_", " ") + } + + return article, err +} + +func extractText(container ast.Container) string { + return string(container.GetChildren()[0].(*ast.Text).Literal) +} + +func ArticleName(path string) string { + ext := filepath.Ext(path) + if ext != ".md" { + return strings.TrimPrefix(path, config.ArticleRoot) + } else { + return strings.TrimSuffix(strings.TrimPrefix(path, config.ArticleRoot), filepath.Ext(path)) + } +} diff --git a/article/notify.go b/article/notify.go new file mode 100644 index 0000000..d9888e3 --- /dev/null +++ b/article/notify.go @@ -0,0 +1,68 @@ +package article + +import ( + "TheAdversary/config" + "TheAdversary/database" + "database/sql" + "fmt" + "github.com/fsnotify/fsnotify" + "sync" + "time" +) + +func Notify() error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + err = watcher.Add(config.ArticleRoot) + if err != nil { + return err + } + db := database.GetDB() + + var lock sync.Mutex + + for { + select { + case event, ok := <-watcher.Events: + lock.Lock() + if !ok { + return fmt.Errorf("failed to catch event") + } + if event.Op&fsnotify.Create == fsnotify.Create || event.Op&fsnotify.Write == fsnotify.Write { + _, err := db.GetArticleByName(ArticleName(event.Name)) + if err != nil && err == sql.ErrNoRows { + article, err := LoadArticle(event.Name) + if err != nil { + return err + } + err = db.AddArticle(article) + if err != nil { + return err + } + } else if err != nil { + return err + } else { + article, err := LoadArticle(event.Name) + if err != nil { + return err + } + article.Modified = time.Now().Unix() + if err = db.UpdateArticle(ArticleName(event.Name), article); err != nil { + return err + } + } + } else if event.Op&fsnotify.Rename == fsnotify.Rename { + if err = db.DeleteArticlesByNames(ArticleName(event.Name)); err != nil { + return err + } + } else if event.Op&fsnotify.Remove == fsnotify.Remove { + if err = db.DeleteArticlesByNames(ArticleName(event.Name)); err != nil { + return err + } + } + lock.Unlock() + } + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..140dad7 --- /dev/null +++ b/config/config.go @@ -0,0 +1,24 @@ +package config + +import ( + "github.com/joho/godotenv" + "os" +) + +var ( + ArticleRoot string + + ServerPort string + + DatabaseFile string +) + +func init() { + godotenv.Load() + + ArticleRoot = os.Getenv("ARTICLE_ROOT") + + ServerPort = os.Getenv("SERVER_PORT") + + DatabaseFile = os.Getenv("DATABASE_FILE") +} diff --git a/database.sqlite3 b/database.sqlite3 new file mode 100644 index 0000000..1101968 Binary files /dev/null and b/database.sqlite3 differ diff --git a/database/article.go b/database/article.go new file mode 100644 index 0000000..806d8cb --- /dev/null +++ b/database/article.go @@ -0,0 +1,94 @@ +package database + +import ( + "TheAdversary/schema" + "database/sql" + "fmt" +) + +type ArticleQueryOptions struct { + Name bool + Title bool + Summary bool + From int64 + To int64 + Limit int +} + +func (db *Database) GetArticleByName(name string) (*schema.Article, error) { + article := &schema.Article{} + err := db.gormDB.Table("article").Where("name = ?", name).Scan(article).Error + if article.Added == 0 { + return nil, sql.ErrNoRows + } + return article, err +} + +func (db *Database) GetArticles(query string, options ArticleQueryOptions) ([]*schema.Article, error) { + request := db.gormDB.Table("article") + var where bool + if options.Name { + if where { + request.Or("name LIKE ?", fmt.Sprintf("%%%s%%", query)) + } else { + request.Where("name LIKE ?", fmt.Sprintf("%%%s%%", query)) + where = true + } + } + if options.Title { + if where { + request.Or("title LIKE ?", fmt.Sprintf("%%%s%%", query)) + } else { + request.Where("title LIKE ?", fmt.Sprintf("%%%s%%", query)) + where = true + } + } + if options.Summary { + if where { + request.Or("summary LIKE ?", fmt.Sprintf("%%%s%%", query)) + } else { + request.Where("summary LIKE ?", fmt.Sprintf("%%%s%%", query)) + where = true + } + } + if !(options.From == 0 || options.To == 0) { + var from, to int64 + if options.From != 0 { + from = options.From + } + if options.To != 0 { + to = options.To + } + request.Where("added BETWEEN ? AND ?", from, to) + } + if options.Limit > 0 { + request.Limit(options.Limit) + } + rows, err := request.Rows() + if err != nil { + return nil, err + } + + var articles []*schema.Article + for rows.Next() { + article := &schema.Article{} + if err = db.gormDB.ScanRows(rows, article); err != nil { + return nil, err + } + articles = append(articles, article) + } + + return articles, nil +} + +func (db *Database) AddArticle(article *schema.Article) error { + return db.gormDB.Table("article").Create(article).Error +} + +func (db *Database) UpdateArticle(name string, article *schema.Article) error { + return db.gormDB.Table("article").Where("name = ?", name).Save(article).Error +} + +func (db *Database) DeleteArticlesByNames(names ...string) error { + return db.gormDB.Table("article").Where("name IN (?)", names).Delete("*").Error +} diff --git a/database/database.go b/database/database.go new file mode 100644 index 0000000..1b73c80 --- /dev/null +++ b/database/database.go @@ -0,0 +1,33 @@ +package database + +import ( + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +var globDatabase *Database + +type Database struct { + gormDB *gorm.DB +} + +func newDatabaseConnection(dialector gorm.Dialector) (*Database, error) { + db, err := gorm.Open(dialector) + if err != nil { + return nil, err + } + + return &Database{db}, nil +} + +func NewSqlite3Connection(databaseFile string) (*Database, error) { + return newDatabaseConnection(sqlite.Open(databaseFile)) +} + +func GetDB() *Database { + return globDatabase +} + +func SetGlobDB(database *Database) { + globDatabase = database +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..47e6004 --- /dev/null +++ b/go.mod @@ -0,0 +1,19 @@ +module TheAdversary + +go 1.17 + +require ( + github.com/fsnotify/fsnotify v1.4.9 + github.com/gomarkdown/markdown v0.0.0-20211207152620-5d6539fd8bfc + github.com/gorilla/mux v1.8.0 + github.com/joho/godotenv v1.4.0 + gorm.io/driver/sqlite v1.2.6 + gorm.io/gorm v1.22.4 +) + +require ( + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.3 // indirect + github.com/mattn/go-sqlite3 v1.14.9 // indirect + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f7ec99b --- /dev/null +++ b/go.sum @@ -0,0 +1,34 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gomarkdown/markdown v0.0.0-20211207152620-5d6539fd8bfc h1:mmMAHzJGtMsCaDyRgPNMO6cbSzeKCZxHTA1Sn/wirko= +github.com/gomarkdown/markdown v0.0.0-20211207152620-5d6539fd8bfc/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.3 h1:PlHq1bSCSZL9K0wUhbm2pGLoTWs2GwVhsP6emvGV/ZI= +github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= +github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.2.6 h1:SStaH/b+280M7C8vXeZLz/zo9cLQmIGwwj3cSj7p6l4= +gorm.io/driver/sqlite v1.2.6/go.mod h1:gyoX0vHiiwi0g49tv+x2E7l8ksauLK0U/gShcdUsjWY= +gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= +gorm.io/gorm v1.22.4 h1:8aPcyEJhY0MAt8aY6Dc524Pn+pO29K+ydu+e/cXSpQM= +gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= diff --git a/main.go b/main.go new file mode 100644 index 0000000..1bfb30c --- /dev/null +++ b/main.go @@ -0,0 +1,113 @@ +package main + +import ( + "TheAdversary/api" + "TheAdversary/article" + "TheAdversary/config" + "TheAdversary/database" + "TheAdversary/schema" + "TheAdversary/server" + "fmt" + "github.com/gorilla/mux" + "io/fs" + "net/http" + "path/filepath" +) + +func main() { + r := mux.NewRouter() + + r.HandleFunc("/api/recent", api.Recent).Methods(http.MethodGet) + r.HandleFunc("/api/search", api.Search).Methods(http.MethodGet) + + r.HandleFunc("/article/{article}", server.Article).Methods(http.MethodGet) + + db, err := database.NewSqlite3Connection(config.DatabaseFile) + if err != nil { + panic(err) + } + database.SetGlobDB(db) + + if err = checkArticles(); err != nil { + panic(err) + } + + go func() { + if err := article.Notify(); err != nil { + panic(err) + } + }() + + if err := http.ListenAndServe(fmt.Sprintf(":%s", config.ServerPort), r); err != nil { + panic(err) + } +} + +func checkArticles() error { + var files []string + err := filepath.Walk(config.ArticleRoot, func(path string, info fs.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, filepath.Join(config.ArticleRoot, path)) + } + return err + }) + if err != nil { + return err + } + + articles, err := database.GetDB().GetArticles("", database.ArticleQueryOptions{}) + if err != nil { + return err + } + + toAdd, toRemove := checkFiles(files, articles) + + db := database.GetDB() + + for _, addFile := range toAdd { + a, err := article.LoadArticle(addFile) + if err != nil { + return err + } + err = db.AddArticle(a) + if err != nil { + return err + } + } + + if len(toRemove) > 0 { + if err = db.DeleteArticlesByNames(toRemove...); err != nil { + return err + } + } + + return nil +} + +func checkFiles(files []string, articles []*schema.Article) ([]string, []string) { + toAdd := files + for i, file := range files { + articleName := article.ArticleName(file) + for _, a := range articles { + if articleName == a.Name { + toAdd = append(toAdd[:i], toAdd[i+1:]...) + } + } + } + + var toRemove []string + for _, a := range articles { + var found bool + for _, file := range files { + if a.Name == article.ArticleName(file) { + found = true + break + } + } + if !found { + toRemove = append(toRemove, a.Name) + } + } + + return toAdd, toRemove +} diff --git a/parse/parse.go b/parse/parse.go new file mode 100644 index 0000000..982258f --- /dev/null +++ b/parse/parse.go @@ -0,0 +1,41 @@ +package parse + +import ( + "TheAdversary/config" + "TheAdversary/schema" + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/ast" + "github.com/gomarkdown/markdown/html" + "github.com/gomarkdown/markdown/parser" + "os" +) + +func newParser() *parser.Parser { + extensions := parser.CommonExtensions | parser.AutoHeadingIDs + + return parser.NewWithExtensions(extensions) +} + +func newHtmlRenderer(title string) *html.Renderer { + renderOpts := html.RendererOptions{ + Title: title, + Flags: html.CommonFlags | html.CompletePage, + } + return html.NewRenderer(renderOpts) +} + +func Parse(path string) (ast.Node, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return markdown.Parse(raw, newParser()), nil +} + +func ParseToHtml(article *schema.Article) ([]byte, error) { + node, err := Parse(article.FilePath(config.ArticleRoot)) + if err != nil { + return nil, err + } + return markdown.Render(node, newHtmlRenderer(article.Title)), nil +} diff --git a/schema/schema.go b/schema/schema.go new file mode 100644 index 0000000..fb78d94 --- /dev/null +++ b/schema/schema.go @@ -0,0 +1,35 @@ +package schema + +import ( + "fmt" + "path/filepath" +) + +type Article struct { + Name string + Title string + Summary string + Image string + Added int64 + Modified int64 +} + +func (a Article) FilePath(articleDir string) string { + return filepath.Join(articleDir, a.Name) + ".md" +} + +func (a Article) ToArticleSummary() ArticleSummary { + return ArticleSummary{ + Title: a.Title, + Summary: a.Summary, + Image: a.Image, + Link: fmt.Sprintf("/article/%s", a.Name), + } +} + +type ArticleSummary struct { + Title string `json:"title"` + Summary string `json:"summary"` + Image string `json:"image"` + Link string `json:"link"` +} diff --git a/server/article.go b/server/article.go new file mode 100644 index 0000000..5bd1032 --- /dev/null +++ b/server/article.go @@ -0,0 +1,34 @@ +package server + +import ( + "TheAdversary/database" + "TheAdversary/parse" + "database/sql" + "fmt" + "github.com/gorilla/mux" + "net/http" +) + +func Article(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + + articleName := mux.Vars(r)["article"] + + article, err := database.GetDB().GetArticleByName(articleName) + if err != nil { + if err == sql.ErrNoRows { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusInternalServerError) + } + return + } + + parsed, _ := parse.ParseToHtml(article) + if len(parsed) > 0 { + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, string(parsed)) + } else { + w.WriteHeader(http.StatusNotFound) + } +}