tools: Migrate remaining tools to gobin

Replace ./scripts/install_tool.sh with `gobin`, such that we have
consistent handling for all tools needed for build and consistent
versioning within ./tools/mod/go.mod.

Side changes:
  - Expose /scripts/fix.sh that fixes formatting and bom across modules
  - Expose *.sh variants of scripts like build and ./test (first step
towards replacement).
  - Make stderr output of commands explicit and make commands use
different color than callouts.

scripts/fix.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+source ./scripts/test_lib.sh
+source ./scripts/updatebom.sh
+
+function mod_tidy_fix {
+  run rm ./go.sum || return 2
+  run go mod tidy || return 2
+}
+
+log_callout -e "\nFixing etcd code for you...\n"
+
+run_for_modules run go fmt || exit 2
+run_for_modules mod_tidy_fix || exit 2
+run_for_module tests bom_fix || exit 2
+
+log_success -e "\nSUCCESS: etcd code is fixed :)"

scripts/install_tool.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-# Usage ./install_tool.sh {go_module}
-#
-# Install given tool and makes it available on $PATH (assuming standard config),
-# without modification to vendor or go.mod file.
-#
-# When https://github.com/golang/go/issues/40276 is implemented, usage
-# of this script should get replaced by pure: 
-# 
-# go install {go_module}@latest
-# 
-set -e
-
->&2 echo "installing: ${1}"
-(
-  cd "$(mktemp -d)"
-  GO111MODULE=on go get "$1"
-)

scripts/test_lib.sh

@@ -1,6 +1,11 @@
 #!/usr/bin/env bash
 
-ETCD_ROOT_DIR="$(pwd)"
+if [[ "$(go list)" != "go.etcd.io/etcd/v3" ]]; then
+	echo "must be run from 'go.etcd.io/etcd/v3' module directory"
+	exit 255
+fi
+
+ETCD_ROOT_DIR=$(go list -f '{{.Dir}}' "go.etcd.io/etcd/v3")
 
 ####   Convenient IO methods #####
 
@@ -8,11 +13,13 @@ COLOR_RED='\033[0;31m'
 COLOR_ORANGE='\033[0;33m'
 COLOR_GREEN='\033[0;32m'
 COLOR_LIGHTCYAN='\033[0;36m'
-
+COLOR_BLUE='\033[0;94m'
+COLOR_MAGENTA='\033[95m'
+COLOR_BOLD='\033[1m'
 COLOR_NONE='\033[0m' # No Color
 
 function log_error {
-  >&2 echo -n -e "${COLOR_RED}"
+  >&2 echo -n -e "${COLOR_BOLD}${COLOR_RED}"
   >&2 echo "$@"
   >&2 echo -n -e "${COLOR_NONE}"
 }
@@ -29,6 +36,12 @@ function log_callout {
   >&2 echo -n -e "${COLOR_NONE}"
 }
 
+function log_cmd {
+  >&2 echo -n -e "${COLOR_BLUE}"
+  >&2 echo "$@"
+  >&2 echo -n -e "${COLOR_NONE}"
+}
+
 function log_success {
   >&2 echo -n -e "${COLOR_GREEN}"
   >&2 echo "$@"
@@ -70,8 +83,8 @@ function run {
     repro="${command[*]}"
   fi
 
-  log_callout "% ${repro}"
-  "${@}" 2> >(while read -r line; do echo -e "\e[01;31m$line\e[0m" >&2; done)
+  log_cmd "% ${repro}"
+  "${@}" 2> >(while read -r line; do echo -e "stderr: ${COLOR_MAGENTA}${line}${COLOR_NONE}" >&2; done)
   local error_code=$?
   if [ ${error_code} -ne 0 ]; then
     log_error -e "FAIL: (code:${error_code}):\n  % ${repro}"
@@ -87,7 +100,7 @@ function run_for_module {
   local module=${1:-"."}
   shift 1
   (
-    cd "${module}" && "$@"
+    cd "${ETCD_ROOT_DIR}/${module}" && "$@"
   )
 }
 
@@ -211,13 +224,15 @@ function tool_exists {
 
 # tool_get_bin [tool] - returns absolute path to a tool binary (or returns error)
 function tool_get_bin {
-  tool_exists "gobin" "GO111MODULE=off go get -u github.com/myitcv/gobin" || return 2
+  tool_exists "gobin" "GO111MODULE=off go get github.com/myitcv/gobin" || return 2
 
   local tool="$1"
   if [[ "$tool" == *"@"* ]]; then
-    run gobin -p "${tool}" || return 2
+    # shellcheck disable=SC2086
+    run gobin ${GOBINARGS} -p "${tool}" || return 2
   else
-    run_for_module ./tools/mod run gobin -p -m --mod=readonly "${tool}" || return 2
+    # shellcheck disable=SC2086
+    run_for_module ./tools/mod run gobin ${GOBINARGS} -p -m --mod=readonly "${tool}" || return 2
   fi
 }
 
@@ -237,3 +252,5 @@ function run_go_tool {
   run "${cmdbin}" "$@" || return 2
 }
 
+# Ensure gobin is available, as it runs majority of the tools
+run env GO111MODULE=off go get github.com/myitcv/gobin

scripts/updatebom.sh

@@ -1,17 +1,38 @@
 #!/usr/bin/env bash
 
 set -e
+source ./scripts/test_lib.sh
 
-if ! [[ "$0" =~ scripts/updatebom.sh ]]; then
-	echo "must be run from repository root"
-	exit 255
-fi
+function bom_fixlet {
+  log_callout "generating bill-of-materials.json"
 
-./scripts/install_tool.sh github.com/coreos/license-bill-of-materials
+  cp go.mod go.mod.tmp
+  cp go.sum go.sum.tmp
 
-echo "generating bill-of-materials.json"
-license-bill-of-materials \
-    --override-file ./bill-of-materials.override.json \
-    go.etcd.io/etcd/v3 go.etcd.io/etcd/v3/etcdctl > bill-of-materials.json
+  local modules
+  # shellcheck disable=SC2207
+  modules=($(modules_exp))
 
-echo "generated bill-of-materials.json"
+  if GOFLAGS=-mod=mod run_go_tool "github.com/coreos/license-bill-of-materials" \
+      --override-file ../bill-of-materials.override.json \
+      "${modules[@]}" > ../bill-of-materials.json.tmp; then
+    cp ../bill-of-materials.json.tmp ../bill-of-materials.json
+    log_success "bom refreshed"
+  else
+    log_error "FAIL: bom refreshing failed"
+  fi
+  mv go.mod.tmp go.mod
+  mv go.sum.tmp go.sum
+}
+
+function bom_fix {
+  # We regenerate bom from the tests directory, as it's a module
+  # that depends on all other modules, so we can generate comprehensive content.
+  # TODO: Migrate to root module, when root module depends on everything (including server & tests).
+  run_for_module "./tests" bom_fixlet
+}
+
+# only build when called directly, not sourced
+if [[ "$0" =~ updatebom.sh$ ]]; then
+  bom_fix
+fi