From 63298870c360fba11b65be8023362a9e9f6b813b Mon Sep 17 00:00:00 2001 From: beastieboy Date: Wed, 2 Nov 2022 21:44:49 +0000 Subject: [PATCH] Reworked logs, added config file (#31) Reworked error logs: clearer, trimmed Added Info logs Added configuration file Closes #30 Closes #5 Co-authored-by: Nicolas Herry Reviewed-on: https://www.beastieboy.net/git/beastieboy/marmotte/pulls/31 --- cmd/marmotte/main.go | 63 ++++++++++++++++++++++++++++++++++++++++-- configs/marmotte.yaml | 34 +++++++++++++++++++++++ gopher/files.go | 23 +++++++-------- gopher/predicates.go | 14 +++++++--- gopher/request.go | 15 ++++++---- gopher/response.go | 30 ++++++++++---------- gopher/server.go | 21 +++++++++----- gopher/transformers.go | 12 ++++---- 8 files changed, 161 insertions(+), 51 deletions(-) create mode 100644 configs/marmotte.yaml diff --git a/cmd/marmotte/main.go b/cmd/marmotte/main.go index ee145bc..0166267 100644 --- a/cmd/marmotte/main.go +++ b/cmd/marmotte/main.go @@ -1,11 +1,16 @@ package main import ( + "beastieboy/marmotte/gopher" + "fmt" "os" + "path/filepath" + "strings" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/rs/zerolog/log" - "beastieboy/marmotte/gopher" ) func main() { @@ -18,6 +23,8 @@ func main() { viper.BindEnv("logfile") viper.BindEnv("users_gopherspace") viper.BindEnv("errors") + viper.BindEnv("debug") + viper.BindEnv("configfile") viper.SetDefault("host", "localhost") viper.SetDefault("port", 7070) @@ -26,6 +33,8 @@ func main() { viper.SetDefault("logfile", "/var/log/marmotte.log") viper.SetDefault("users_gopherspace", "/users") viper.SetDefault("errors", "/errors") + viper.SetDefault("debug", false) + viper.SetDefault("configfile", "/usr/local/etc/marmotte/marmotte.yaml") pflag.String("host", "localhost", "the hostname the server sockets will bind to") pflag.Int("port", 7070, "the port the server will listen on") @@ -33,11 +42,37 @@ func main() { pflag.String("root", "/usr/local/marmotte", "the root directory where the documents can be found") pflag.String("logfile", "/var/log/marmotte.log", "the logfile (not rotated)") pflag.String("users_gopherspace", "/users", "the root directory for users gopherspace (~username)") - pflag.String("errors", "$ROOT/errors", "the directory where the error templates can be found") + pflag.String("errors", "/errors", "the directory where the error templates can be found") + pflag.Bool("debug", false, "set logging level to Debug") + pflag.String("configfile", "/usr/local/etc/marmotte/marmotte.yaml", "YAML configuration file") pflag.Parse() viper.BindPFlags(pflag.CommandLine) + configfilename:= strings.TrimSuffix(filepath.Base(viper.GetString("configfile")), + filepath.Ext(viper.GetString("configfile"))) + viper.SetConfigName(configfilename) + viper.SetConfigType("yaml") + configdir := filepath.Dir(viper.GetString("configfile")) + viper.AddConfigPath(configdir) + + if err := viper.ReadInConfig(); err != nil { + fmt.Println(fmt.Errorf("wtf config file: %w", err)) + + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + fmt.Printf("Could not find configuration file %s / %s (%s); relying on command-line and defaults.\n", configfilename, configdir, viper.GetString("configfile")) + } else { + panic(fmt.Errorf("fatal error config file: %w", err)) + } + } + + debug := viper.GetBool("debug") + // Default level for this example is info, unless debug flag is present + zerolog.SetGlobalLevel(zerolog.InfoLevel) + if debug { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + } + f, err := os.OpenFile(viper.GetString("logfile"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660) defer f.Close() @@ -48,6 +83,28 @@ func main() { Err(err). Msg("Could not open log file") } + + log.Info(). + Msg("Starting marmotte") + log.Info(). + Str("Root", viper.GetString("root")). + Send() + log.Info(). + Str("Host", viper.GetString("host")). + Send() + log.Info(). + Str("Port", viper.GetString("port")). + Send() + log.Info(). + Str("Transport", viper.GetString("transport")). + Send() + log.Info(). + Str("UsersGopherspace", viper.GetString("users_gopherspace")). + Send() + log.Info(). + Str("ErrorRoot", viper.GetString("error")). + Send() + ctx := &gopher.Context{ Root: viper.GetString("root"), Host: viper.GetString("host"), diff --git a/configs/marmotte.yaml b/configs/marmotte.yaml new file mode 100644 index 0000000..822a897 --- /dev/null +++ b/configs/marmotte.yaml @@ -0,0 +1,34 @@ +# Example configuration file for marmotte +# Copy and edit to suits your needs + +# Host the sockets will be bound to +host: localhost + +# Port the sockets will be bound to +# Note: On Unix-like systems, to listen to ports below 1024 a process must run as a privileged user +port: 7070 + +# Network transport protocol the server will create the sockets with +# Today, only tcp makes sense here +transport: tcp + +# The directory where gopher resources can be found +# The directory must be readable by the user running the server +root: /usr/local/marmotte + +# When not shipping logs to an aggregator, set this to a writable file +# To print on stdout and/or use a logshipper, comment out the following line +logfile: /var/log/marmotte + +# The directory alias for users' gopherspaces +# The path is relative to the root directory +users_gopherspace: /users + +# The path to the directory containing the error templates (10.gt, 20.gt, 30.gt, etc.) +# The path is relative to the root directory +errors: /errors + +# Whether log level should be set to debug +debug: false + + diff --git a/gopher/files.go b/gopher/files.go index 9efcd53..a4d3108 100644 --- a/gopher/files.go +++ b/gopher/files.go @@ -21,25 +21,24 @@ func FileType(p string) (ResponseType, error) { regexp.MustCompile(".*/pdf"): PDF, regexp.MustCompile("(.*/msword)|(application/vnd.openxmlformats-officedocument.wordprocessingml.document)"): Doc, } - log.Debug().Str("p", p).Send() - fi, err := os.Stat(p) if err != nil { log.Error(). Err(err). - Msg("Received an error when trying to stat a full path") + Str("Path", p). + Msg("Received an error when trying to Stat a path") return Binary, err } if fi.IsDir() { return Directory, nil } else { - mtype, err := mimetype.DetectFile(p) if err != nil { - log.Error().Err(err). + log.Error(). + Err(err). Str("Path", p). - Msg("Error while detecting a file type") + Msg("Error while detecting a MIME file type") } for m, t := range mimegopher { @@ -48,7 +47,7 @@ func FileType(p string) (ResponseType, error) { Str("Filetype", mtype.String()). Str("Extension", mtype.Extension()). Str("regexp", m.String()). - Send() + Msg("Matched a file against a MIME file type") return t, nil } @@ -77,7 +76,8 @@ func GopherMap(c Context, r string) ([]string, error) { // build a dynamic gophermap listing the contents of the directory entries, err := os.ReadDir(c.Root + r) if err != nil { - log.Error().Err(err). + log.Error(). + Err(err). Str("Path", r). Msg("Error reading the contents of a directory") return []string{}, err @@ -111,7 +111,6 @@ func GopherMap(c Context, r string) ([]string, error) { Msg("Error reading the contents of a gophermap") return []string{}, err } else { - return GopherMapFixLines(c, lines), nil } } @@ -147,7 +146,10 @@ func ReadTextFile(p string) ([]string, error) { lines := []string{} f, err := os.Open(p) defer f.Close() - + + log.Debug(). + Str("Path", p). + Msg("Reading text file") if err != nil { return []string{}, fmt.Errorf("Error reading the file %s: %s", p, err.Error()) } @@ -159,6 +161,5 @@ func ReadTextFile(p string) ([]string, error) { lines = append(lines, fs.Text() + "\r\n") } - log.Debug().Msg("Done reading file") return lines, nil } diff --git a/gopher/predicates.go b/gopher/predicates.go index 877f6c5..b420d9c 100644 --- a/gopher/predicates.go +++ b/gopher/predicates.go @@ -14,13 +14,19 @@ type TransformerPredicate func(Context, Request, Response, error) bool func FullPathMatchPredicateFor(p string) func(Context, Request, Response, error)(bool) { return func(c Context, rq Request, rs Response, e error) bool { r, e := regexp.Compile(p) + log.Debug(). + Str("Path", rq.FullPath). + Str("Pattern", p). + Msg("Trying to match a path against a pattern") + if e == nil && r.MatchString(rq.FullPath) { return true } else if e != nil { - log.Err(e). - Str("fullpath", rq.FullPath). - Str("pattern", p). - Send() + log.Error(). + Err(e). + Str("Path", rq.FullPath). + Str("Pattern", p). + Msg("Error while trying to match a path against a pattern") } return false diff --git a/gopher/request.go b/gopher/request.go index bde57a4..392b5de 100644 --- a/gopher/request.go +++ b/gopher/request.go @@ -34,7 +34,8 @@ func GetRequestTypeFor(c Context, s string) RequestType { if err != nil { log.Error(). Err(err). - Msg("Received an error when trying to stat a full path") + Str("Selector", s). + Msg("Selector cannot be stat as a file, RequestType set to Unknown") return Unknown } if fi.IsDir() { @@ -47,11 +48,7 @@ func GetRequestTypeFor(c Context, s string) RequestType { // NewRequest creates a new Request instance and returns a pointer to it. It relies on the selector s to determine the RequestType, and on the Context c to build the paths and verify the files. func NewRequest(c Context, s string) (*Request, error) { r := Request{} - log.Debug(). - Str("s", s). - Send() ContentList := regexp.MustCompile("^$") - // Selector := regexp.MustCompile("^(.+)$") if ContentList.MatchString(s) { r.Path = "/" @@ -62,5 +59,13 @@ func NewRequest(c Context, s string) (*Request, error) { r.FullPath = c.Root + r.Path r.Type = GetRequestTypeFor(c, s) } + + log.Debug(). + Str("Selector", s). + Str("FullPath", r.FullPath). + Str("Path", r.Path). + Int("RequestType", int(r.Type)). + Msg("New request") + return &r, nil } diff --git a/gopher/response.go b/gopher/response.go index 0deb11e..49f6e6a 100644 --- a/gopher/response.go +++ b/gopher/response.go @@ -2,7 +2,7 @@ package gopher import ( "github.com/rs/zerolog/log" - "github.com/gabriel-vasile/mimetype" + // "github.com/gabriel-vasile/mimetype" ) // The type ResponseType describes the possible types of Responses: PlainText, Audio, Image, PDF, Doc, Binary and Directory, as specified by the Gopher Protocol. @@ -52,21 +52,17 @@ func NewResponse(c Context, rq Request) *Response { if rq.Type == Map { r.Type = PlainText } else { - mtype, err := mimetype.DetectFile(rq.FullPath) - if err != nil { - log.Error(). - Err(err). - Str("FullPath", rq.FullPath). - Msg("Error detecting a file type") - return &r - } + // mtype, err := mimetype.DetectFile(rq.FullPath) + // if err != nil { + // log.Error(). + // Err(err). + // Str("FullPath", rq.FullPath). + // Msg("Error detecting a MIME file type") + // return &r + // } - log.Debug(). - Str("Filetype:", mtype.String()). - Str("Extension", mtype.Extension()). - Send() t, err := FileType(rq.FullPath) - log.Debug().Int("t", int(t)).Send() + if err != nil { log.Error(). Err(err). @@ -77,6 +73,12 @@ func NewResponse(c Context, rq Request) *Response { r.Type = t } + + log.Debug(). + Str("FullPath", rq.FullPath). + Int("ResponseType", int(r.Type)). + Msg("New Response") + return &r } diff --git a/gopher/server.go b/gopher/server.go index b7cabaa..8685bfb 100644 --- a/gopher/server.go +++ b/gopher/server.go @@ -21,6 +21,12 @@ func write(ctx *Context, rq Request, rs *Response, w io.Writer) error { } else { w.Write(rs.ContentBinary) } + + // later: log from inet.addr, etc. + log.Info(). + Str("Request", rq.FullPath). + Msg("Done serving response") + return nil } @@ -37,22 +43,23 @@ func ParseRequest(c *Context, r io.Reader) (*Request, error) { if e := s.Err(); e != nil { log.Error(). Err(e). - Msg("Error reading input") + Msg("Error reading input data when parsing Request") return &Request{}, e } + + // later: log from inet.addr, etc. + log.Info(). + Str("Selector", s.Text()). + Msg("Incoming selector request") rq, err := NewRequest(*c, s.Text()) if err != nil { log.Error(). Err(err). - Msg("Got an error when creating a new Request: ") + Msg("Got an error when creating a new Request") } - log.Debug(). - Str("Request", rq.Path). - Send() - return rq, err } @@ -104,7 +111,7 @@ func HandleConnection(ctx *Context, c net.Conn) error { if err != nil { log.Error(). Err(err). - Msg("An error occurred while writing the data on the socket:") + Msg("An error occurred while writing the data on the socket") } c.Close() // always close connection diff --git a/gopher/transformers.go b/gopher/transformers.go index 9911a88..fd57938 100644 --- a/gopher/transformers.go +++ b/gopher/transformers.go @@ -68,7 +68,7 @@ func BinaryFileTransformer(c *Context, rq Request, rs *Response, e error) (*Cont if err != nil { log.Error(). Err(err). - Str("Path", rq.FullPath). + Str("FullPath", rq.FullPath). Msg("Error reading a binary file") return c, rq, rs, err } @@ -124,7 +124,8 @@ func SelectorRewriteTransformerFor(from string, to string) func(*Context, *Reque if err != nil { log.Error(). Err(err). - Msg("Error compiling a regexp for a SelectorRewriteTransformer: %s") + Str("RegexpFrom", from). + Msg("Error compiling a regexp for a SelectorRewriteTransformer") return c, rq, rs, err } @@ -158,7 +159,7 @@ func ExecutableTransformer(c *Context, rq Request, rs *Response, err error) (*Co log.Error(). Err(err). Str("Executable", rq.FullPath). - Msg("An error occurred creating a Request for the executable") + Msg("An error occurred creating a Request for an executable") } } // by default, return parameters unchanged @@ -180,16 +181,13 @@ func ErrorRedirectTransformer(c *Context, rq Request, rs *Response, err error) ( if t != nil { var buf bytes.Buffer err = t.Execute(io.Writer(&buf), GopherErrorInfo { Selector: rq.Path }) - log.Debug(). - Str("Buffer", buf.String()). - Send() rs.ContentText = []string{buf.String()} rs.Type = PlainText return c, rq, rs, err } log.Error(). - Str("Error template path", p). + Str("Path", p). Msg("Error reading an error template") rs.ContentText = []string {fmt.Sprint("Error retrieving the error page for the request. The original error code was ", rs.ErrorCode, " for the selector ", rq.Path)}