Browse Source

Add static linking of extra plugins, other updates

master
Tyler Sommer 1 year ago
parent
commit
4a00f26c1f
Signed by: tyler-sommer
GPG Key ID: C09C010500DBD008
  1. 3
      .gitignore
  2. 24
      Dockerfile
  3. 71
      Makefile
  4. 19
      README.md
  5. 92
      cli/flags.go
  6. 64
      cli/manager.go
  7. 18
      cmd/squircy/linked_plugins.go
  8. 114
      cmd/squircy/main.go
  9. 9
      cmd/squircy/shared_plugins.go
  10. 13
      config.toml.dist
  11. 44
      config/option.go
  12. 56
      config/option_flags_test.go
  13. 2
      config/option_map_test.go
  14. 115
      config/option_post.go
  15. 2
      config/plugin.go
  16. 51
      config/setup.go
  17. 63
      config/value.go
  18. 10
      event/event.go
  19. 4
      event/plugin.go
  20. 14
      go.mod
  21. 35
      go.sum
  22. 30
      irc/irc.go
  23. 13
      irc/plugin.go
  24. 2
      package.json
  25. 14
      plugin/plugin.go
  26. 12
      plugins/babel/plugin.go
  27. 14
      plugins/babel/plugin/main.go
  28. 4
      plugins/node_compat/child_process.go
  29. 4
      plugins/node_compat/crypto.go
  30. 4
      plugins/node_compat/event_emitter.go
  31. 4
      plugins/node_compat/exec.go
  32. 4
      plugins/node_compat/exec_test.go
  33. 4
      plugins/node_compat/http.go
  34. 4
      plugins/node_compat/net.go
  35. 4
      plugins/node_compat/net_test.go
  36. 6
      plugins/node_compat/plugin.go
  37. 14
      plugins/node_compat/plugin/main.go
  38. 4
      plugins/node_compat/stream.go
  39. 14
      plugins/script/plugin.go
  40. 15
      plugins/script/plugin/main.go
  41. 2
      plugins/script/script.go
  42. 6
      plugins/squircy2_compat/helper.go
  43. 16
      plugins/squircy2_compat/plugin.go
  44. 14
      plugins/squircy2_compat/plugin/main.go
  45. 2
      plugins/squircy2_compat/runtime.go
  46. 11
      vm/plugin.go
  47. 68
      vm/require.go
  48. 3
      vm/result.go
  49. 7
      vm/vm.go

3
.gitignore vendored

@ -2,8 +2,7 @@
!.gitignore
!.dockerignore
out/*
node_modules/*
testdata/node_modules/*
node_modules
testdata/yarn-error.log
testdata/yarn.lock
yarn.lock

24
Dockerfile

@ -1,6 +1,10 @@
FROM golang:buster AS build
FROM golang:alpine AS build
ARG race
ARG plugin_type=shared
RUN apk update && \
apk add yarn alpine-sdk upx
WORKDIR /squircy
@ -11,19 +15,13 @@ RUN go get -v github.com/gobuffalo/packr/v2/... && \
RUN go get -v ./...
RUN make clean all RACE=${race}
RUN make clean dist RACE=${race} PLUGIN_TYPE=${plugin_type}
FROM debian:buster-slim
RUN apt-get update && \
apt-get upgrade -y && \
apt-get install -y ca-certificates curl gnupg
FROM alpine:latest
RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
apt-get update && \
apt-get install -y yarn
RUN apk update && \
apk add ca-certificates curl gnupg yarn
COPY config.toml.dist /home/squircy/.squircy/config.toml
@ -32,7 +30,7 @@ COPY package.json /home/squircy/.squircy/scripts/package.json
RUN cd /home/squircy/.squircy/scripts && \
yarn install
RUN useradd -d /home/squircy squircy
RUN adduser -D -h /home/squircy squircy
RUN chown -R squircy: /home/squircy
@ -40,7 +38,7 @@ USER squircy
WORKDIR /squircy
COPY --from=build /squircy/out/squircy /bin/squircy
COPY --from=build /squircy/out/squircy_linux_amd64 /bin/squircy
COPY --from=build /squircy/out/*.so /squircy/plugins/

71
Makefile

@ -3,36 +3,61 @@
SUBPACKAGES := cli config event irc plugin vm
PLUGINS := $(patsubst plugins/%,%,$(wildcard plugins/*))
PLUGINS ?= $(patsubst plugins/%,%,$(wildcard plugins/*))
SOURCES := $(wildcard cmd/*/*.go) $(wildcard $(patsubst %,%/*.go,$(SUBPACKAGES)))
GENERATOR_SOURCES := $(wildcard cmd/squircy/defconf/*)
OUTPUT_BASE := out
PLUGIN_TARGETS := $(patsubst %,$(OUTPUT_BASE)/%.so,$(PLUGINS))
SQUIRCY_TARGET := $(OUTPUT_BASE)/squircy
RACE ?= -race
RACE ?= -race
TEST_ARGS ?= -count 1
# Include PLUGIN_TYPE=linked in the command-line when invoking make to link
# extra plugins directly in the main binary rather than generating shared object files.
PLUGIN_TYPE ?= shared
GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH)
GOARM ?= $(shell go env GOARM)
CC ?= $(shell go env CC)
PACKR ?= $(shell which packr || which packr2)
SQUIRCY_TARGET := $(OUTPUT_BASE)/squircy
SQUIRCY_DIST := $(SQUIRCY_TARGET)_$(GOOS)_$(GOARCH)$(GOARM)
PLUGIN_DIST := $(patsubst %,$(OUTPUT_BASE)/%_$(GOOS)_$(GOARCH)$(GOARM).so,$(PLUGINS))
ifeq ($(PLUGIN_TYPE),linked)
PLUGIN_TARGETS :=
EXTRA_TAGS := -tags linked_plugins
DIST_TARGETS := $(SQUIRCY_DIST)
LINKED_PLUGINS_FILE := cmd/squircy/linked_plugins.go
else
PLUGIN_TARGETS := $(patsubst %,$(OUTPUT_BASE)/%.so,$(PLUGINS))
EXTRA_TAGS :=
DIST_TARGETS := $(SQUIRCY_DIST) $(PLUGIN_DIST)
LINKED_PLUGINS_FILE := cmd/squircy/shared_plugins.go
endif
TESTDATA_NODEMODS_TARGET := testdata/node_modules
.PHONY: all build generate run plugins clean test
SQUIRCY3_VERSION := $(if $(shell test -d .git && echo "1"),$(shell git describe --always --tags),SNAPSHOT)
.PHONY: all build run plugins clean test dist
all: build plugins
clean:
cd cmd/squircy && \
packr2 clean
$(PACKR) clean
rm -rf $(OUTPUT_BASE)
build: generate $(SQUIRCY_TARGET)
generate: $(OUTPUT_BASE)/.generated
build: $(SQUIRCY_TARGET)
plugins: $(PLUGIN_TARGETS)
dist: $(DIST_TARGETS)
run: build
$(SQUIRCY_TARGET)
@ -45,19 +70,31 @@ $(TESTDATA_NODEMODS_TARGET):
.SECONDEXPANSION:
$(PLUGIN_TARGETS): $(OUTPUT_BASE)/%.so: $$(wildcard plugins/%/*) $(SOURCES)
go build -tags netgo $(RACE) -o $@ -buildmode=plugin plugins/$*/*.go
go build -tags netgo $(RACE) -o $@ -buildmode=plugin plugins/$*/plugin/*.go
.SECONDEXPANSION:
$(PLUGIN_DIST): $(OUTPUT_BASE)/%_$(GOOS)_$(GOARCH)$(GOARM).so: $$(wildcard plugins/%/*) $(SOURCES)
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) CC=$(CC) CGO_ENABLED=1 \
go build -tags netgo $(EXTRA_TAGS) \
-ldflags "-s -w -X main.Version=$(SQUIRCY3_VERSION)" \
-o $@ -buildmode=plugin plugins/$*/plugin/*.go
$(SQUIRCY_TARGET): $(SOURCES)
go build -tags netgo $(RACE) -o $@ cmd/squircy/*.go
go build -tags netgo $(EXTRA_TAGS) $(RACE) -ldflags "-X main.Version=$(SQUIRCY3_VERSION)-dev" \
-o $@ cmd/squircy/main.go cmd/squircy/repl.go $(LINKED_PLUGINS_FILE)
$(OUTPUT_BASE)/.generated: $(GENERATOR_SOURCES)
$(SQUIRCY_DIST): $(OUTPUT_BASE) $(SOURCES)
cd cmd/squircy/defconf && \
yarn install
cd cmd/squircy && \
packr2
touch $@
$(PACKR)
GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) CC=$(CC) CGO_ENABLED=1 \
go build -tags netgo $(EXTRA_TAGS) \
-ldflags "-s -w -X main.Version=$(SQUIRCY3_VERSION)" \
-o $@ cmd/squircy/main.go cmd/squircy/repl.go $(LINKED_PLUGINS_FILE)
upx $@
$(OUTPUT_BASE):
mkdir -p $(OUTPUT_BASE)
$(SOURCES): $(OUTPUT_BASE)
$(GENERATOR_SOURCES): $(OUTPUT_BASE)

19
README.md

@ -36,18 +36,18 @@ directory as the root.
does not contain config.toml or package.json, defaults will be created.
```bash
out/squircy -interactive -root out
out/squircy -interactive -root out/config
```
This will automatically create `config.toml` and `package.json` within the
`out/` directory. The default configuration enables all plugins available
and connects to freenode. As such, on the first run, expect to see some
`out/config/` directory. The default configuration enables all plugins available
and connects to libera.chat. As such, on the first run, expect to see some
warnings.
> Expect to see some warnings on the first run. All plugins are enabled, by
default, so NodeJS dependencies must be installed for everything to function.
Modify the default configuration in `out/config.toml` as necessary. Comment
Modify the default configuration in `out/config/config.toml` as necessary. Comment
out or remove the plugins listed under `extra_plugins` to disable any
unwanted plugins.
@ -125,6 +125,17 @@ as shared libraries and loaded at runtime using the Go plugin API.
[squircy2](https://squircy.com).
- `script` loads javascript files from a configured folder at app startup.
#### Linking extra plugins at compile-time
squircy3 supports building-in the extra plugins at compile-time so that they
are included in the main binary rather than as separate shared object files.
Pass `PLUGIN_TYPE=linked` to make to enable this functionality.
```bash
make all PLUGIN_TYPE=linked
```
## Related Projects

92
cli/flags.go

@ -0,0 +1,92 @@
package cli
import (
"flag"
"fmt"
"strings"
)
func PluginOptsFlag(fs *flag.FlagSet, name, usage string) {
val := make(pluginOptsFlag)
PluginOptsFlagVar(fs, &val, name, usage)
}
func PluginOptsFlagVar(fs *flag.FlagSet, val flag.Value, name, usage string) {
fs.Var(val, name, usage)
}
type pluginOptsFlag map[string]interface{}
func (s pluginOptsFlag) String() string {
var res []string
for k, v := range s {
res = append(res, fmt.Sprintf("%s=%v", k, v))
}
return strings.Join(res, " ")
}
func (s pluginOptsFlag) Set(str string) error {
p := strings.SplitN(str, "=", 2)
if len(p) == 1 {
p = append(p, "true")
}
var v interface{}
if p[1] == "true" {
v = true
} else if p[1] == "false" {
v = false
} else {
v = p[1]
}
s[p[0]] = v
return nil
}
func (s pluginOptsFlag) Get() interface{} {
return map[string]interface{}(s)
}
type stringsFlag []string
func (s stringsFlag) String() string {
return strings.Join(s, "")
}
func (s *stringsFlag) Set(str string) error {
*s = append(*s, str)
return nil
}
func (s stringsFlag) Get() interface{} {
return []string(s)
}
func DefaultFlags(fs *flag.FlagSet) {
CoreFlags(fs, "~/.squircy")
IRCFlags(fs)
VMFlags(fs)
}
func CoreFlags(fs *flag.FlagSet, root string) {
extraPlugins := stringsFlag{}
fs.String("root", root, "path to folder containing application data")
fs.String("log-level", "info", "controls verbosity of logging output")
fs.Var(&extraPlugins, "plugin", "path to shared plugin .so file, multiple plugins may be given")
PluginOptsFlag(fs, "plugin-option", "specify extra plugin configuration option, format: key=value")
}
func IRCFlags(fs *flag.FlagSet) {
fs.Bool("irc-auto", false, "automatically connect to irc")
fs.String("irc-nick", "squishyjones", "specify irc nickname")
fs.String("irc-user", "mrjones", "specify irc user")
fs.String("irc-network", "chat.freenode.net:6697", "specify irc network")
fs.Bool("irc-tls", true, "use tls encryption when connecting to irc")
fs.Bool("irc-sasl", false, "use sasl authentication")
fs.String("irc-sasl-username", "", "specify sasl username")
fs.String("irc-sasl-password", "", "specify sasl password")
fs.String("irc-server-password", "", "specify server password")
}
func VMFlags(fs *flag.FlagSet) {
fs.String("vm-modules-path", "node_modules", "specify javascript modules path")
}

64
cli/manager.go

@ -1,5 +1,5 @@
// import "code.dopame.me/veonik/squircy3/cli"
package cli
// Package cli makes reusable the core parts of the squircy command.
package cli // import "code.dopame.me/veonik/squircy3/cli"
import (
"flag"
@ -8,9 +8,9 @@ import (
"path/filepath"
"syscall"
"github.com/sirupsen/logrus"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
tilde "gopkg.in/mattes/go-expand-tilde.v1"
"code.dopame.me/veonik/squircy3/config"
"code.dopame.me/veonik/squircy3/event"
@ -20,13 +20,13 @@ import (
)
type Config struct {
RootDir string `toml:"root_path",flag:"root"`
RootDir string `toml:"root_path" flag:"root"`
PluginDir string `toml:"plugin_path"`
ExtraPlugins []string `toml:"extra_plugins"`
ExtraPlugins []string `toml:"extra_plugins" flag:"plugin"`
PluginOptions map[string]interface{}
PluginOptions map[string]interface{} `flag:"plugin_option"`
LogLevel string `toml:"log_level"`
LogLevel logrus.Level `toml:"log_level"`
// Specify additional plugins that are a part of the main executable.
LinkedPlugins []plugin.Initializer
@ -40,7 +40,7 @@ type Manager struct {
stop chan os.Signal
}
func NewManager(rootDir string, pluginOptions map[string]interface{}, extraPlugins ...string) (*Manager, error) {
func NewManager() (*Manager, error) {
m := plugin.NewManager()
// initialize only the config plugin so that it can be configured before
// other plugins are initialized
@ -48,22 +48,47 @@ func NewManager(rootDir string, pluginOptions map[string]interface{}, extraPlugi
if err := configure(m); err != nil {
return nil, err
}
conf := Config{
RootDir: rootDir,
PluginDir: filepath.Join(rootDir, "plugins"),
ExtraPlugins: extraPlugins,
PluginOptions: pluginOptions,
}
conf := Config{}
// configure the config plugin!
cf := filepath.Join(rootDir, "config.toml")
err := config.ConfigurePlugin(m,
config.WithRequiredOptions("root_path", "log_level"),
config.WithFilteredOption("root_path", func(s string, val config.Value) (config.Value, error) {
vs, ok := val.(string)
if !ok {
return nil, errors.Errorf("expected root_path to be string but got %T", vs)
}
vs, err := tilde.Expand(vs)
if err != nil {
return nil, errors.Errorf("failed to expand root directory: %s", err)
}
return vs, nil
}),
config.WithFilteredOption("log_level", func(s string, val config.Value) (config.Value, error) {
if v, ok := val.(logrus.Level); ok {
return v, nil
}
vs, ok := val.(string)
if !ok {
return nil, errors.Errorf("expected log_level to be string but got %T", vs)
}
lvl, err := logrus.ParseLevel(vs)
if err != nil {
lvl = logrus.InfoLevel
logrus.Warnf("config: defaulting to info log level: failed to parse %s as log level: %s", lvl, err)
}
return lvl, nil
}),
config.WithInitValue(&conf),
config.WithValuesFromTOMLFile(cf),
config.WithValuesFromFlagSet(flag.CommandLine),
config.WithValuesFromMap(conf.PluginOptions))
config.WithValuesFromMap(&conf.PluginOptions))
if err != nil {
return nil, err
}
cf := filepath.Join(conf.RootDir, "config.toml")
// Now that we have determined the root directory
if err := config.ConfigurePlugin(m, config.WithValuesFromTOMLFile(cf)); err != nil {
return nil, err
}
return &Manager{
plugins: m,
stop: make(chan os.Signal, 10),
@ -97,6 +122,7 @@ func (manager *Manager) Start() error {
// load remaining extra plugins
for _, pl := range manager.ExtraPlugins {
logrus.Tracef("core: loading extra plugin: %s", pl)
if !filepath.IsAbs(pl) {
pl = filepath.Join(manager.PluginDir, pl)
}
@ -162,7 +188,7 @@ func (manager *Manager) Loop() error {
case syscall.SIGTERM:
manager.Stop()
default:
logrus.Debugln("Received signal", s, "but not doing anything with it")
logrus.Debugln("core: received signal", s, "but not doing anything with it")
}
}
}

18
cmd/squircy/linked_plugins.go

@ -0,0 +1,18 @@
// +build linked_plugins
package main
import (
"code.dopame.me/veonik/squircy3/plugin"
babel "code.dopame.me/veonik/squircy3/plugins/babel"
node_compat2 "code.dopame.me/veonik/squircy3/plugins/node_compat"
script "code.dopame.me/veonik/squircy3/plugins/script"
squircy2_compat "code.dopame.me/veonik/squircy3/plugins/squircy2_compat"
)
var linkedPlugins = []plugin.Initializer{
plugin.InitializerFunc(babel.Initialize),
plugin.InitializerFunc(node_compat2.Initialize),
plugin.InitializerFunc(script.Initialize),
plugin.InitializerFunc(squircy2_compat.Initialize)}

114
cmd/squircy/main.go

@ -6,82 +6,35 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/gobuffalo/packr/v2"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
tilde "gopkg.in/mattes/go-expand-tilde.v1"
"code.dopame.me/veonik/squircy3/cli"
"code.dopame.me/veonik/squircy3/irc"
)
type stringsFlag []string
var Version = "SNAPSHOT"
func (s stringsFlag) String() string {
return strings.Join(s, "")
}
func (s *stringsFlag) Set(str string) error {
*s = append(*s, str)
return nil
}
type pluginOptsFlag map[string]interface{}
func (s pluginOptsFlag) String() string {
var res []string
for k, v := range s {
res = append(res, fmt.Sprintf("%s=%s", k, v))
}
return strings.Join(res, " ")
}
func (s pluginOptsFlag) Set(str string) error {
p := strings.SplitN(str, "=", 2)
if len(p) == 1 {
p = append(p, "true")
}
var v interface{}
if p[1] == "true" {
v = true
} else if p[1] == "false" {
v = false
} else {
v = p[1]
}
s[p[0]] = v
return nil
}
type stringLevel logrus.Level
func (s stringLevel) String() string {
return logrus.Level(s).String()
}
func (s *stringLevel) Set(str string) error {
l, err := logrus.ParseLevel(str)
if err != nil {
return err
}
*s = stringLevel(l)
return nil
}
var rootDir string
var extraPlugins stringsFlag
var pluginOptions = make(pluginOptsFlag)
var logLevel = stringLevel(logrus.InfoLevel)
var interactive bool
func unboxAll(rootDir string) error {
box := packr.New("defconf", "./defconf")
if _, err := os.Stat(rootDir); err == nil {
// root directory already exists, don't muck with it
return nil
}
if err := os.MkdirAll(rootDir, 0755); err != nil {
return errors.Wrap(err, "failed to create root directory")
}
box := packr.New("defconf", "./defconf")
for _, f := range box.List() {
dst := filepath.Join(rootDir, f)
if _, err := os.Stat(dst); os.IsNotExist(err) {
logrus.Infof("Creating default %s as %s", f, dst)
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return errors.Wrap(err, "failed to recreate directory")
}
logrus.Infof("Creating default %s", dst)
d, err := box.Find(f)
if err != nil {
return errors.Wrapf(err, "failed to get contents of boxed %s", f)
@ -95,21 +48,10 @@ func unboxAll(rootDir string) error {
}
func init() {
flag.StringVar(&rootDir, "root", "~/.squircy", "path to folder containing squircy data")
flag.Var(&logLevel, "log-level", "controls verbosity of logging output")
flag.Var(&extraPlugins, "plugin", "path to shared plugin .so file, multiple plugins may be given")
printVersion := false
flag.BoolVar(&interactive, "interactive", false, "start interactive-read-evaluate-print (REPL) session")
flag.Var(&pluginOptions, "plugin-option", "specify extra plugin configuration option, format: key=value")
flag.Bool("irc-auto", false, "automatically connect to irc")
flag.String("irc-nick", "squishyjones", "specify irc nickname")
flag.String("irc-user", "mrjones", "specify irc user")
flag.String("irc-network", "chat.freenode.net:6697", "specify irc network")
flag.Bool("irc-tls", true, "use tls encryption when connecting to irc")
flag.Bool("irc-sasl", false, "use sasl authentication")
flag.String("irc-sasl-username", "", "specify sasl username")
flag.String("irc-sasl-password", "", "specify sasl password")
flag.String("irc-server-password", "", "specify server password")
flag.String("vm-modules-path", "node_modules", "specify javascript modules path")
flag.BoolVar(&printVersion, "version", false, "print version information")
cli.DefaultFlags(flag.CommandLine)
flag.Usage = func() {
fmt.Println("Usage: ", os.Args[0], "[options]")
@ -120,28 +62,34 @@ func init() {
flag.PrintDefaults()
}
flag.Parse()
var err error
if rootDir, err = tilde.Expand(rootDir); err != nil {
logrus.Fatalln("core: failed determine root directory:", err)
}
if err := unboxAll(rootDir); err != nil {
logrus.Fatalln("core: failed to unbox defaults:", err)
if printVersion {
fmt.Printf("squircy3 %s\n", Version)
os.Exit(0)
}
}
func main() {
logrus.SetLevel(logrus.Level(logLevel))
m, err := cli.NewManager(rootDir, pluginOptions, extraPlugins...)
logrus.SetLevel(logrus.InfoLevel)
m, err := cli.NewManager()
if err != nil {
logrus.Fatalln("core: error initializing squircy:", err)
}
m.LinkedPlugins = append(m.LinkedPlugins, linkedPlugins...)
logrus.SetLevel(m.LogLevel)
if err := unboxAll(m.RootDir); err != nil {
logrus.Fatalln("core: failed to unbox defaults:", err)
}
if err := m.Start(); err != nil {
logrus.Fatalln("core: error starting squircy:", err)
}
ircm, err := irc.FromPlugins(m.Plugins())
if err != nil {
logrus.Errorln("core: failed to set irc version string:", err)
}
ircm.SetVersionString(fmt.Sprintf("squircy3 %s", Version))
if interactive {
go func() {
Repl(m)
}()
go Repl(m)
}
if err = m.Loop(); err != nil {
logrus.Fatalln("core: exiting main loop with error:", err)

9
cmd/squircy/shared_plugins.go

@ -0,0 +1,9 @@
// +build !linked_plugins
package main
import (
"code.dopame.me/veonik/squircy3/plugin"
)
var linkedPlugins []plugin.Initializer

13
config.toml.dist

@ -1,8 +1,18 @@
plugin_path="/squircy/plugins"
extra_plugins=[
# babel seamlessly transpiles javascript before executing it in squircy3, enabling the use of
# ES2017+ features in your scripts.
"babel.so",
# node_compat is a barebones compatibility layer for NodeJS APIs. It supports a limited set of
# features such as parts of child_process, net, and crypto. Requires babel.
"node_compat.so",
# squircy2_compat is a compatibility layer for legacy squIRCy2 scripts.
"squircy2_compat.so",
# script is a plugin that loads scripts from a directory and executes them during application
# startup.
"script.so",
]
@ -26,12 +36,15 @@ enable=true
scripts_path="/home/squircy/.squircy/scripts"
[squircy2_compat]
# set enable_file_api to true to allow scripts to read from the filesystem.
enable_file_api=false
# set the root directory of the filesystem access; files outside this root will not be loaded.
file_api_root="/home/squircy/.squircy/files"
#owner_nick=""
#owner_host=""
data_path="/home/squircy/.squircy/data"
[node_compat]
# set enable_exec to true to allow scripts to spawn child processes.
enable_exec=false

44
config/option.go

@ -63,6 +63,7 @@ func WithSection(sec Section, options ...SetupOption) SetupOption {
return err
}
s.sections[ns.name] = ns
s.sectionsOrdered = append(s.sectionsOrdered, ns)
return nil
}
}
@ -106,12 +107,47 @@ func WithOption(name string) SetupOption {
func WithOptions(names ...string) SetupOption {
return func(s *Setup) error {
for _, n := range names {
s.options[n] = false
if _, ok := s.options[n]; !ok {
s.options[n] = nil
s.optionsOrdered = append(s.optionsOrdered, n)
}
}
return nil
}
}
// WithValidatedOption adds a value validator for the given option.
// Validator functions accept the name of the option and its value, and
// return an error if the value is not considered valid.
func WithValidatedOption(name string, fn func(string, Value) error) SetupOption {
return func(s *Setup) error {
s.options[name] = append(s.options[name], fn)
return nil
}
}
// WithFilteredOption adds a filter that may modify the option's value.
// Filters are applied after
func WithFilteredOption(name string, fn func(string, Value) (Value, error)) SetupOption {
return func(s *Setup) error {
s.filters[name] = append(s.filters[name], fn)
return nil
}
}
// ValidateRequired is a validator that ensures an option is not blank.
// Any nil value or string with length of zero is considered blank.
func ValidateRequired(o string, v Value) error {
var nilValue Value
if v == nil || v == nilValue {
return errors.Errorf(`required option "%s" is empty`, o)
}
if vs, ok := v.(string); ok && len(vs) == 0 {
return errors.Errorf(`required option "%s" is empty`, o)
}
return nil
}
// WithRequiredOption adds a required option to the Config.
func WithRequiredOption(name string) SetupOption {
return WithRequiredOptions(name)
@ -121,7 +157,9 @@ func WithRequiredOption(name string) SetupOption {
func WithRequiredOptions(names ...string) SetupOption {
return func(s *Setup) error {
for _, n := range names {
s.options[n] = true
if err := WithValidatedOption(n, ValidateRequired)(s); err != nil {
return errors.Wrapf(err, "validation failed for %s", n)
}
}
return nil
}
@ -134,7 +172,7 @@ func WithInheritedOption(name string) SetupOption {
if ps == nil {
return errors.Errorf("config: unable to inherit option '%s' for section %s; no parent found", name, s.name)
}
s.inherits[name] = struct{}{}
s.inherits = append(s.inherits, name)
return nil
}
}

56
config/option_flags_test.go

@ -5,6 +5,8 @@ import (
"fmt"
"testing"
"github.com/sirupsen/logrus"
"code.dopame.me/veonik/squircy3/config"
)
@ -18,7 +20,7 @@ func TestWithValuesFromFlagSet(t *testing.T) {
co := &Config{"veonik", &struct{ Age int }{30}}
fs := flag.NewFlagSet("", flag.ExitOnError)
fs.String("name", "", "your name")
fs.Int("bio-age", 0, "you age")
fs.Int("bio-age", 0, "your age")
if err := fs.Parse([]string{"-name", "tyler", "-bio-age", "31"}); err != nil {
t.Errorf("unexpected error parsing flagset: %s", err)
return
@ -58,3 +60,55 @@ func TestWithValuesFromFlagSet(t *testing.T) {
// Hi, tyler!
// tyler is 31.
}
func TestWithValuesFromFlagSetDefaultValues(t *testing.T) {
logrus.SetLevel(logrus.TraceLevel)
co := map[string]interface{}{}
fs := flag.NewFlagSet("", flag.ExitOnError)
fs.String("name", "veonik", "your name")
fs.Int("bio-age", 31, "your age")
if err := fs.Parse([]string{}); err != nil {
t.Errorf("unexpected error parsing flagset: %s", err)
return
}
c, err := config.Wrap(co,
config.WithRequiredOption("name"),
config.WithGenericSection("bio", config.WithRequiredOption("age")),
config.WithValuesFromFlagSet(fs))
if err != nil {
t.Errorf("expected config to be valid, but got error: %s", err)
return
}
fmt.Printf("Hi, %s!\n", co["name"])
n, ok := c.String("name")
if !ok {
t.Errorf("expected name to be a string")
return
}
b, err := c.Section("bio")
if err != nil {
t.Errorf("expected config to be valid, but got error: %s", err)
return
}
a, ok := b.Int("age")
if !ok {
t.Errorf("expected age to be an int")
return
}
if co["name"] != n {
t.Errorf("expected Name option (%s) to match Name field on Config struct (%s)", n, co["name"])
return
}
bio, ok := co["bio"].(map[string]interface{})
if !ok {
t.Errorf("expected map to contain another map with key bio, but got: %v", bio)
return
}
if bio["age"] != a {
t.Errorf("expected bio.age option (%d) to match Age field on Bio struct (%d)", a, bio["age"])
}
fmt.Printf("%s is %d.\n", n, a)
// Outputs:
// Hi, tyler!
// tyler is 31.
}

2
config/option_map_test.go

@ -24,7 +24,7 @@ func TestWithValuesFromMap(t *testing.T) {
c, err := config.Wrap(co,
config.WithRequiredOption("Name"),
config.WithGenericSection("Bio", config.WithRequiredOption("Age")),
config.WithValuesFromMap(opts))
config.WithValuesFromMap(&opts))
if err != nil {
t.Errorf("expected config to be valid, but got error: %s", err)
return

115
config/option_post.go

@ -2,6 +2,7 @@ package config
import (
"flag"
"os"
"regexp"
"strings"
@ -18,6 +19,10 @@ func populateValuesFromTOMLFile(filename string) postSetupOption {
s.raw = make(map[string]interface{})
}
if _, err := toml.DecodeFile(filename, &s.raw); err != nil {
if os.IsNotExist(err) {
logrus.Warnf("config: unable to load toml config, but not aborting: %s", err)
return nil
}
return err
}
return nil
@ -28,7 +33,7 @@ func populateValuesFromTOMLFile(filename string) postSetupOption {
// TOML file.
func WithValuesFromTOMLFile(filename string) SetupOption {
return func(s *Setup) error {
return s.addPostSetup(populateValuesFromTOMLFile(filename))
return s.prependPostSetup(populateValuesFromTOMLFile(filename))
}
}
@ -45,10 +50,10 @@ func newNameFieldMapper(s *Setup) *nameFieldMapper {
return &nameFieldMapper{s}
}
// normalize converts the given name into a normal, underscorized name.
// normalizeName converts the given name into a normal, underscorized name.
// Dashes are converted to underscores, camel case is separated by underscores
// and converts everything to lower-case.
func (fm *nameFieldMapper) normalize(name string) string {
func normalizeName(name string) string {
name = dashAndSpaceMatcher.ReplaceAllString(name, "_")
var a []string
for _, sub := range camelCaseMatcher.FindAllStringSubmatch(name, -1) {
@ -69,45 +74,47 @@ func (fm *nameFieldMapper) normalize(name string) string {
// is converted into the path:
// ["Templating","Twig","views_path"]
func (fm *nameFieldMapper) Map(flagName string) (path []string) {
normal := fm.normalize(flagName)
normal := normalizeName(flagName)
s := fm.s
loop:
for s != nil {
// a valueInspector here handles struct tag aliases.
is, err := inspect(s.initial)
if err != nil {
logrus.Debugf("config: unable to create valueInspector for section '%s': %s", s.name, err)
} else {
if _, err := is.Get(normal); err == nil {
path = append(path, normal)
logrus.Debugf("config: valueInspector found match for field '%s' in section '%s'", normal, s.name)
s = nil
// check for a matching section, using the name as a prefix
for k, ks := range s.sections {
kn := normalizeName(k) + "_"
if strings.HasPrefix(normal, kn) {
// found the next step in the path
normal = strings.Replace(normal, kn, "", 1)
path = append(path, k)
logrus.Tracef("config: descending into section %s (from %s) to find match for option '%s'", ks.name, s.name, normal)
s = ks
goto loop
} else {
logrus.Debugf("config: valueInspector returned error for '%s' in section '%s': %s", normal, s.name, err)
}
}
// check for a match in options next
for k := range s.options {
kn := fm.normalize(k)
kn := normalizeName(k)
if kn == normal {
// found it
path = append(path, k)
logrus.Debugf("config: found option with name '%s' in section '%s'", normal, s.name)
logrus.Tracef("config: found option with name '%s' in section '%s'", normal, s.name)
s = nil
goto loop
}
}
// check for a matching section, using the name as a prefix
for k, ks := range s.sections {
kn := fm.normalize(k) + "_"
if strings.HasPrefix(normal, kn) {
// found the next step in the path
normal = strings.Replace(normal, kn, "", 1)
path = append(path, k)
logrus.Debugf("config: descending into section %s (from %s) to find match for option '%s'", ks.name, s.name, normal)
s = ks
// finally, try a valueInspector here to handle struct tag aliases.
is, err := inspect(s.initial)
if err != nil {
logrus.Debugf("config: unable to create valueInspector for section '%s': %s", s.name, err)
} else {
if name, err := is.Normalize(normal); err == nil {
path = append(path, name)
logrus.Tracef("config: valueInspector found match for field '%s' in section '%s'", name, s.name)
s = nil
goto loop
} else {
if !strings.Contains(err.Error(), "no field with name") {
logrus.Debugf("config: valueInspector returned error for '%s' in section '%s': %s", normal, s.name, err)
}
}
}
return nil
@ -115,16 +122,15 @@ loop:
return path
}
func visitNamedOption(s *Setup, f string, fv interface{}, m *nameFieldMapper) {
func visitNamedOption(name string, raw map[string]interface{}, f string, fv interface{}, m *nameFieldMapper, override bool) {
path := m.Map(f)
if len(path) == 0 {
logrus.Debugf("config: did not match anything for named option '%s' for section %s", f, s.name)
logrus.Debugf("config: did not match anything for named option '%s' for section %s", f, name)
return
}
logrus.Debugf("config: named option '%s' mapped to path: %v", f, path)
logrus.Debugf("config: named option '%s' setting to: %T(%v)", f, fv, fv)
logrus.Debugf("config: named option '%s' mapped to path %v and value %T(%v)", f, path, fv, fv)
val := fv
v := s.raw
v := raw
i := 0
// iterate over all but the last part of the path, descending into a
// new section with each iteration.
@ -133,8 +139,13 @@ func visitNamedOption(s *Setup, f string, fv interface{}, m *nameFieldMapper) {
v = vs
} else {
if vr, ok := v[path[i]]; ok {
if vrn, ok := vr.(map[string]interface{}); ok {
// value exists and is already a map[string]interface{}, use it.
v = vrn
continue
}
// there is no path[0-1] so figure out the name accordingly
secn := s.name
secn := name
if i > 0 {
secn = path[i-1]
}
@ -145,25 +156,26 @@ func visitNamedOption(s *Setup, f string, fv interface{}, m *nameFieldMapper) {
v = nv
}
}
if !override {
if _, ok := v[path[i]]; ok {
// value is already set, don't override it.
return
}
}
// use the last element in the path to set the right option.
v[path[i]] = val
}
func populateValuesFromFlagSet(fs *flag.FlagSet) postSetupOption {
return func(s *Setup) error {
if !fs.Parsed() {
return errors.Errorf("given FlagSet must be parsed")
}
if s.raw == nil {
s.raw = make(map[string]interface{})
}
logrus.Tracef("config: populating values from FlagSet %s", fs.Name())
m := newNameFieldMapper(s)
fs.Visit(func(f *flag.Flag) {
var v interface{} = f.Value.String()
if fv, ok := f.Value.(flag.Getter); ok {
v = fv.Get()
}
visitNamedOption(s, f.Name, v, m)
visitNamedOption(s.name, s.raw, f.Name, v, m, true)
})
return nil
}
@ -175,26 +187,39 @@ func WithValuesFromFlagSet(fs *flag.FlagSet) SetupOption {
if !fs.Parsed() {
return errors.Errorf("given FlagSet must be parsed")
}
return s.addPostSetup(populateValuesFromFlagSet(fs))
if s.raw == nil {
s.raw = make(map[string]interface{})
}
m := newNameFieldMapper(s)
fs.VisitAll(func(f *flag.Flag) {
var v interface{} = f.Value.String()
if fv, ok := f.Value.(flag.Getter); ok {
v = fv.Get()
}
visitNamedOption(s.name, s.raw, f.Name, v, m, false)
})
return s.appendPostSetup(populateValuesFromFlagSet(fs))
}
}
func populateValuesFromMap(vs map[string]interface{}) postSetupOption {
func populateValuesFromMap(vs *map[string]interface{}) postSetupOption {
return func(s *Setup) error {
logrus.Tracef("config: populating values from map: %p", vs)
if s.raw == nil {
s.raw = make(map[string]interface{})
}
m := newNameFieldMapper(s)
for f, fv := range vs {
visitNamedOption(s, f, fv, m)
for f, fv := range *vs {
visitNamedOption(s.name, s.raw, f, fv, m, true)
}
return nil
}
}
// WithValuesFromMap populates the Config using the given map.
func WithValuesFromMap(vs map[string]interface{}) SetupOption {
func WithValuesFromMap(vs *map[string]interface{}) SetupOption {
return func(s *Setup) error {
return s.addPostSetup(populateValuesFromMap(vs))
return s.appendPostSetup(populateValuesFromMap(vs))
}
}

2
config/plugin.go

@ -18,6 +18,7 @@ func pluginFromPlugins(m *plugin.Manager) (*configPlugin, error) {
return mp, nil
}
// ConfigurePlugin applies the given options to the registered config plugin.
func ConfigurePlugin(m *plugin.Manager, opts ...SetupOption) error {
mp, err := pluginFromPlugins(m)
if err != nil {
@ -26,6 +27,7 @@ func ConfigurePlugin(m *plugin.Manager, opts ...SetupOption) error {
return mp.Configure(opts...)
}
// Initialize is a plugin.Initializer that initializes a config plugin.
func Initialize(m *plugin.Manager) (plugin.Plugin, error) {
p := &configPlugin{}
return p, nil

51
config/setup.go

@ -1,8 +1,6 @@
package config
import (
"fmt"
"github.com/pkg/errors"
)
@ -19,6 +17,12 @@ type postSetupOption func(c *Setup) error
// A protoFunc is a function that returns the initial value for a config.
type protoFunc func() Value
// An optionFilter modifies the received value and returns the result.
type optionFilter func(name string, val Value) (Value, error)
// An optionValidator returns an error if the given value is invalid.
type optionValidator func(name string, val Value) error
// Setup is a container struct with information on how to setup a given Config.
type Setup struct {
name string
@ -31,9 +35,12 @@ type Setup struct {
raw map[string]interface{}
sections map[string]*Setup
options map[string]bool
inherits map[string]struct{}
sectionsOrdered []*Setup
sections map[string]*Setup
optionsOrdered []string
options map[string][]optionValidator
filters map[string][]optionFilter
inherits []string
post []postSetupOption
}
@ -48,17 +55,23 @@ func newSetup(name string, parent *Setup) *Setup {
parent: parent,
raw: make(map[string]interface{}),
sections: make(map[string]*Setup),
options: make(map[string]bool),
inherits: make(map[string]struct{}),
options: make(map[string][]optionValidator),
filters: make(map[string][]optionFilter),
}
}
// addPostSetup adds one or more postSetupOptions.
func (s *Setup) addPostSetup(options ...postSetupOption) error {
// appendPostSetup adds one or more postSetupOptions to the end.
func (s *Setup) appendPostSetup(options ...postSetupOption) error {
s.post = append(s.post, options...)
return nil
}
// prependPostSetup adds one or more postSetupOptions to the beginning.
func (s *Setup) prependPostSetup(options ...postSetupOption) error {
s.post = append(append([]postSetupOption{}, options...), s.post...)
return nil
}
// apply calls each SetupOption, halting on the first error encountered.
func (s *Setup) apply(options ...SetupOption) error {
// clear post options, they will be re-added by the regular options.
@ -83,15 +96,11 @@ func (s *Setup) validate() error {
if s.config == nil {
return errors.New(`expected config to be populated, found nil`)
}
for o, reqd := range s.options {
if reqd {
var nilValue Value
v, ok := s.config.Get(o)
if !ok || v == nil || v == nilValue {
return errors.Errorf(`required option "%s" is empty`, o)
}
if vs, ok := v.(string); ok && len(vs) == 0 {
return errors.Errorf(`required option "%s" is empty`, o)
for o, ovs := range s.options {
for _, validator := range ovs {
v, _ := s.config.Get(o)
if err := validator(o, v); err != nil {
return errors.Wrapf(err, `failed to validate option "%s"`, o)
}
}
}
@ -107,7 +116,7 @@ func (s *Setup) validate() error {
func walkAndWrap(s *Setup) error {
wrapErr := func(err error) error {
if s.name != "root" {
return errors.WithMessage(err, fmt.Sprintf("section %s", s.name))
return errors.WithMessage(err, "section "+s.name)
}
return err
}
@ -142,7 +151,7 @@ func walkAndWrap(s *Setup) error {
// walkInherits synchronizes inherited options and sections between this and
// the parent.
func walkInherits(s *Setup) error {
for si := range s.inherits {
for _, si := range s.inherits {
if s.parent == nil {
return errors.Errorf("unable to inherit option %s from non-existent parent", si)
}
@ -161,7 +170,7 @@ func walkInherits(s *Setup) error {
// walkSections walks through each section, populating a Config for each.
func walkSections(s *Setup) error {
for _, ns := range s.sections {
for _, ns := range s.sectionsOrdered {
if v, ok := s.raw[ns.name].(map[string]interface{}); ok {
ns.raw = v
}

63
config/value.go

@ -1,7 +1,6 @@
package config
import (
"fmt"
"reflect"
"github.com/fatih/structtag"
@ -19,6 +18,8 @@ type configurable struct {
Value
options configurableOpts
inspector *valueInspector
filters map[string][]optionFilter
}
func newConfigurable(s *Setup) (*configurable, error) {
@ -29,11 +30,12 @@ func newConfigurable(s *Setup) (*configurable, error) {
if err != nil {
return nil, err
}
c := &configurable{Value: s.initial, options: make(configurableOpts), inspector: is}
c := &configurable{Value: s.initial, options: make(configurableOpts), filters: s.filters, inspector: is}
for k, v := range s.raw {
logrus.Tracef("Setting %s.%s to %T(%v)", s.name, k, v, v)
c.Set(k, v)
}
for k := range s.options {
for _, k := range s.optionsOrdered {
v, err := c.inspector.Get(k)
if err != nil {
return nil, err
@ -95,9 +97,16 @@ func (c *configurable) Int(key string) (int, bool) {
func (c *configurable) Set(key string, val Value) {
defer func() {
if err := recover(); err != nil {
fmt.Println(err)
logrus.Errorf("config: unexpected panic while trying to set option %s: %s", key, err)
}
}()
for _, filter := range c.filters[key] {
if nv, err := filter(key, val); err != nil {
logrus.Errorf("config: error while filtering value for option %s: %s", key, err)
} else {
val = nv
}
}
c.options[key] = val
c.inspector.Set(key, val)
}
@ -147,6 +156,10 @@ func inspect(v Value) (*valueInspector, error) {
}, nil
}
func (i *valueInspector) Normalize(name string) (string, error) {
return i.realName(name)
}
func (i *valueInspector) Get(name string) (Value, error) {
return i.valueNamed(name)
}
@ -177,6 +190,11 @@ func (i *valueInspector) Set(name string, val Value) {
res = append(res, vs)
}
}
} else if v, ok := val.([]string); ok {
res = v
} else {
logrus.Warnf("config: unsupported value type for slice: %T", val)
return
}
rv = reflect.ValueOf(res)
default: