From 87b16abd93ff60acd245776d5b0d914fd580c259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 5 Apr 2019 10:09:22 +0200 Subject: [PATCH] Add HUGO_NUMWORKERMULTIPLIER And use that to calculate number of workers, if set, else fall back to number of logical CPUs. Also tweak the relevant related settings to match the new setup. Also remove the setting of `runtime.GOMAXPROCS` as this has been the default behaviour since Go 1.5. Fixes #5814 --- config/env.go | 33 +++++++++++++++++++ .../en/getting-started/configuration.md | 5 +++ hugolib/pagebundler.go | 8 ++--- hugolib/pagebundler_capture.go | 8 ++--- hugolib/site.go | 9 ----- hugolib/site_render.go | 4 ++- main.go | 4 --- 7 files changed, 47 insertions(+), 24 deletions(-) create mode 100644 config/env.go diff --git a/config/env.go b/config/env.go new file mode 100644 index 000000000..adf6f9b68 --- /dev/null +++ b/config/env.go @@ -0,0 +1,33 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "runtime" + "strconv" +) + +// GetNumWorkerMultiplier returns the base value used to calculate the number +// of workers to use for Hugo's parallel execution. +// It returns the value in HUGO_NUMWORKERMULTIPLIER OS env variable if set to a +// positive integer, else the number of logical CPUs. +func GetNumWorkerMultiplier() int { + if gmp := os.Getenv("HUGO_NUMWORKERMULTIPLIER"); gmp != "" { + if p, err := strconv.Atoi(gmp); err == nil && p > 0 { + return p + } + } + return runtime.NumCPU() +} diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index 5b310db15..4ef42465f 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -297,6 +297,11 @@ enableemoji: true ``` {{% /note %}} +## Configuration Environment Variables + +HUGO_NUMWORKERMULTIPLIER +: Can be set to increase or reduce the number of workers used in parallel processing in Hugo. If not set, the number of logical CPUs will be used. + ## Configuration Lookup Order Similar to the template [lookup order][], Hugo has a default set of rules for searching for a configuration file in the root of your website's source directory as a default behavior: diff --git a/hugolib/pagebundler.go b/hugolib/pagebundler.go index 682221d8c..5149968bc 100644 --- a/hugolib/pagebundler.go +++ b/hugolib/pagebundler.go @@ -18,7 +18,8 @@ import ( "fmt" "math" "path/filepath" - "runtime" + + "github.com/gohugoio/hugo/config" _errors "github.com/pkg/errors" @@ -73,10 +74,7 @@ func (s *siteContentProcessor) processAsset(asset pathLangFile) { } func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { - numWorkers := 12 - if n := runtime.NumCPU() * 3; n > numWorkers { - numWorkers = n - } + numWorkers := config.GetNumWorkerMultiplier() * 3 numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.h.Sites)))) diff --git a/hugolib/pagebundler_capture.go b/hugolib/pagebundler_capture.go index 17a4b865a..7c01a751d 100644 --- a/hugolib/pagebundler_capture.go +++ b/hugolib/pagebundler_capture.go @@ -19,7 +19,8 @@ import ( "os" "path" "path/filepath" - "runtime" + + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/common/loggers" _errors "github.com/pkg/errors" @@ -70,10 +71,7 @@ func newCapturer( contentChanges *contentChangeMap, filenames ...string) *capturer { - numWorkers := 4 - if n := runtime.NumCPU(); n > numWorkers { - numWorkers = n - } + numWorkers := config.GetNumWorkerMultiplier() // TODO(bep) the "index" vs "_index" check/strings should be moved in one place. isBundleHeader := func(filename string) bool { diff --git a/hugolib/site.go b/hugolib/site.go index 84ee7823c..616b5b37d 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1881,15 +1881,6 @@ func (s *Site) newPage(kind string, sections ...string) *pageState { return p } -func getGoMaxProcs() int { - if gmp := os.Getenv("GOMAXPROCS"); gmp != "" { - if p, err := strconv.Atoi(gmp); err == nil { - return p - } - } - return 1 -} - func (s *Site) shouldBuild(p page.Page) bool { return shouldBuild(s.BuildFuture, s.BuildExpired, s.BuildDrafts, p.Draft(), p.PublishDate(), p.ExpiryDate()) diff --git a/hugolib/site_render.go b/hugolib/site_render.go index f3df09f09..1d8b14b0a 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -19,6 +19,8 @@ import ( "strings" "sync" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/output" "github.com/pkg/errors" @@ -55,7 +57,7 @@ func (s siteRenderContext) renderSingletonPages() bool { // TODO(bep np doc func (s *Site) renderPages(ctx *siteRenderContext) error { - numWorkers := getGoMaxProcs() * 4 + numWorkers := config.GetNumWorkerMultiplier() results := make(chan error) pages := make(chan *pageState, numWorkers) // buffered for performance diff --git a/main.go b/main.go index d04012111..ecb423e60 100644 --- a/main.go +++ b/main.go @@ -14,16 +14,12 @@ package main import ( - "runtime" - "os" "github.com/gohugoio/hugo/commands" ) func main() { - - runtime.GOMAXPROCS(runtime.NumCPU()) resp := commands.Execute(os.Args[1:]) if resp.Err != nil {