1701 lines
68 KiB
Python
1701 lines
68 KiB
Python
# Copyright (c) 2018, 2019 Nordic Semiconductor ASA
|
|
# Copyright 2018, 2019 Foundries.io
|
|
#
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
'''West project commands'''
|
|
|
|
import argparse
|
|
from functools import partial
|
|
import logging
|
|
import os
|
|
from os.path import abspath, relpath, exists
|
|
from pathlib import PurePath, Path
|
|
import shutil
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
from time import perf_counter
|
|
from urllib.parse import urlparse
|
|
|
|
from west.configuration import config, update_config
|
|
from west import log
|
|
from west import util
|
|
from west.commands import WestCommand, CommandError
|
|
from west.manifest import ImportFlag, Manifest, MANIFEST_PROJECT_INDEX, \
|
|
ManifestProject, _manifest_content_at, ManifestImportFailed, \
|
|
_ManifestImportDepth, ManifestVersionError, MalformedManifest
|
|
from west.manifest import is_group as is_project_group
|
|
from west.manifest import MANIFEST_REV_BRANCH as MANIFEST_REV
|
|
from west.manifest import QUAL_MANIFEST_REV_BRANCH as QUAL_MANIFEST_REV
|
|
from west.manifest import QUAL_REFS_WEST as QUAL_REFS
|
|
|
|
#
|
|
# Project-related or multi-repo commands, like "init", "update",
|
|
# "diff", etc.
|
|
#
|
|
|
|
class _ProjectCommand(WestCommand):
|
|
# Helper class which contains common code needed by various commands
|
|
# in this file.
|
|
|
|
def _parser(self, parser_adder, **kwargs):
|
|
# Create and return a "standard" parser.
|
|
|
|
kwargs['help'] = self.help
|
|
kwargs['description'] = self.description
|
|
kwargs['formatter_class'] = argparse.RawDescriptionHelpFormatter
|
|
return parser_adder.add_parser(self.name, **kwargs)
|
|
|
|
def _cloned_projects(self, args, only_active=False):
|
|
# Returns _projects(args.projects, only_cloned=True) if
|
|
# args.projects is not empty (i.e., explicitly given projects
|
|
# are required to be cloned). Otherwise, returns all cloned
|
|
# projects.
|
|
if args.projects:
|
|
ret = self._projects(args.projects, only_cloned=True)
|
|
else:
|
|
ret = [p for p in self.manifest.projects if p.is_cloned()]
|
|
|
|
if args.projects or not only_active:
|
|
return ret
|
|
|
|
return [p for p in ret if self.manifest.is_active(p)]
|
|
|
|
def _projects(self, ids, only_cloned=False):
|
|
try:
|
|
return self.manifest.get_projects(ids, only_cloned=only_cloned)
|
|
except ValueError as ve:
|
|
if len(ve.args) != 2:
|
|
raise # not directly raised by get_projects()
|
|
|
|
# Die with an error message on unknown or uncloned projects.
|
|
unknown, uncloned = ve.args
|
|
if unknown:
|
|
die_unknown(unknown)
|
|
elif only_cloned and uncloned:
|
|
s = 's' if len(uncloned) > 1 else ''
|
|
names = ' '.join(p.name for p in uncloned)
|
|
log.die(f'uncloned project{s}: {names}.\n'
|
|
' Hint: run "west update" and retry.')
|
|
else:
|
|
# Should never happen, but re-raise to fail fast and
|
|
# preserve a stack trace, to encourage a bug report.
|
|
raise
|
|
|
|
def _handle_failed(self, args, failed):
|
|
# Shared code for commands (like status, diff, update) that need
|
|
# to do the same thing to multiple projects, but collect
|
|
# and report errors if anything failed.
|
|
|
|
if not failed:
|
|
return
|
|
elif len(failed) < 20:
|
|
s = 's:' if len(failed) > 1 else ''
|
|
projects = ', '.join(f'{p.name}' for p in failed)
|
|
log.err(f'{self.name} failed for project{s} {projects}')
|
|
else:
|
|
log.err(f'{self.name} failed for multiple projects; see above')
|
|
raise CommandError(1)
|
|
|
|
def _setup_logging(self, args):
|
|
logger = logging.getLogger('west.manifest')
|
|
|
|
verbose = min(args.verbose, log.VERBOSE_EXTREME)
|
|
if verbose >= log.VERBOSE_NORMAL:
|
|
level = logging.DEBUG
|
|
else:
|
|
level = logging.INFO
|
|
|
|
logger.setLevel(level)
|
|
logger.addHandler(ProjectCommandLogHandler())
|
|
|
|
class Init(_ProjectCommand):
|
|
|
|
def __init__(self):
|
|
super().__init__(
|
|
'init',
|
|
'create a west workspace',
|
|
f'''\
|
|
Creates a west workspace.
|
|
|
|
With -l, creates a workspace around an existing local repository;
|
|
without -l, creates a workspace by cloning a manifest repository
|
|
by URL.
|
|
|
|
With -m, clones the repository at that URL and uses it as the
|
|
manifest repository. If --mr is not given, the remote's default
|
|
branch will be used, if it exists.
|
|
|
|
With neither, -m {MANIFEST_URL_DEFAULT} is assumed.
|
|
''',
|
|
requires_workspace=False)
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
# We set a custom usage because there are two distinct ways
|
|
# to call this command, and the default usage generated by
|
|
# argparse doesn't make that very clear.
|
|
|
|
parser = self._parser(
|
|
parser_adder,
|
|
usage='''
|
|
|
|
%(prog)s [-m URL] [--mr REVISION] [--mf FILE] [directory]
|
|
%(prog)s -l [--mf FILE] directory
|
|
''')
|
|
|
|
# Remember to update the usage if you modify any arguments.
|
|
|
|
parser.add_argument('-m', '--manifest-url',
|
|
help='''manifest repository URL to clone;
|
|
cannot be combined with -l''')
|
|
parser.add_argument('--mr', '--manifest-rev', dest='manifest_rev',
|
|
help='''manifest revision to check out and use;
|
|
cannot be combined with -l''')
|
|
parser.add_argument('--mf', '--manifest-file', dest='manifest_file',
|
|
help='manifest file name to use')
|
|
parser.add_argument('-l', '--local', action='store_true',
|
|
help='''use "directory" as an existing local
|
|
manifest repository instead of cloning one from
|
|
MANIFEST_URL; .west is created next to "directory"
|
|
in this case, and manifest.path points at
|
|
"directory"''')
|
|
|
|
parser.add_argument(
|
|
'directory', nargs='?', default=None,
|
|
help='''with -l, the path to the local manifest repository;
|
|
without it, the directory to create the workspace in (defaulting
|
|
to the current working directory in this case)''')
|
|
|
|
return parser
|
|
|
|
def do_run(self, args, _):
|
|
if self.topdir:
|
|
zb = os.environ.get('ZEPHYR_BASE')
|
|
if zb:
|
|
msg = textwrap.dedent(f'''
|
|
Note:
|
|
In your environment, ZEPHYR_BASE is set to:
|
|
{zb}
|
|
|
|
This forces west to search for a workspace there.
|
|
Try unsetting ZEPHYR_BASE and re-running this command.''')
|
|
else:
|
|
msg = ''
|
|
self.die_already(self.topdir, msg)
|
|
|
|
if args.local and (args.manifest_url or args.manifest_rev):
|
|
log.die('-l cannot be combined with -m or --mr')
|
|
|
|
self.die_if_no_git()
|
|
|
|
self._setup_logging(args)
|
|
|
|
if args.local:
|
|
topdir = self.local(args)
|
|
else:
|
|
topdir = self.bootstrap(args)
|
|
|
|
log.banner(f'Initialized. Now run "west update" inside {topdir}.')
|
|
|
|
def die_already(self, where, also=None):
|
|
log.die(f'already initialized in {where}, aborting.{also or ""}')
|
|
|
|
def local(self, args) -> Path:
|
|
if args.manifest_rev is not None:
|
|
log.die('--mr cannot be used with -l')
|
|
|
|
# We need to resolve this to handle the case that args.directory
|
|
# is '.'. In that case, Path('.').parent is just Path('.') instead of
|
|
# Path('..').
|
|
#
|
|
# https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.parent
|
|
manifest_dir = Path(args.directory or os.getcwd()).resolve()
|
|
manifest_filename = args.manifest_file or 'west.yml'
|
|
manifest_file = manifest_dir / manifest_filename
|
|
topdir = manifest_dir.parent
|
|
rel_manifest = manifest_dir.name
|
|
west_dir = topdir / WEST_DIR
|
|
|
|
if not manifest_file.is_file():
|
|
log.die(f'can\'t init: no {manifest_filename} found in '
|
|
f'{manifest_dir}')
|
|
|
|
log.banner('Initializing from existing manifest repository',
|
|
rel_manifest)
|
|
log.small_banner(f'Creating {west_dir} and local configuration file')
|
|
self.create(west_dir)
|
|
os.chdir(topdir)
|
|
update_config('manifest', 'path', os.fspath(rel_manifest))
|
|
update_config('manifest', 'file', manifest_filename, topdir=topdir)
|
|
|
|
return topdir
|
|
|
|
def bootstrap(self, args) -> Path:
|
|
topdir = Path(abspath(args.directory or os.getcwd()))
|
|
log.banner('Initializing in', topdir)
|
|
|
|
manifest_url = args.manifest_url or MANIFEST_URL_DEFAULT
|
|
manifest_rev = args.manifest_rev or self.get_head_branch(manifest_url)
|
|
west_dir = topdir / WEST_DIR
|
|
|
|
try:
|
|
already = util.west_topdir(topdir, fall_back=False)
|
|
self.die_already(already)
|
|
except util.WestNotFound:
|
|
pass
|
|
|
|
if not topdir.is_dir():
|
|
self.create(topdir, exist_ok=False)
|
|
|
|
# Clone the manifest repository into a temporary directory.
|
|
tempdir: Path = west_dir / 'manifest-tmp'
|
|
if tempdir.is_dir():
|
|
log.dbg('removing existing temporary manifest directory', tempdir)
|
|
shutil.rmtree(tempdir)
|
|
try:
|
|
self.clone_manifest(manifest_url, manifest_rev, os.fspath(tempdir))
|
|
except subprocess.CalledProcessError:
|
|
shutil.rmtree(tempdir, ignore_errors=True)
|
|
raise
|
|
|
|
# Verify the manifest file exists.
|
|
temp_manifest_filename = args.manifest_file or 'west.yml'
|
|
temp_manifest = tempdir / temp_manifest_filename
|
|
if not temp_manifest.is_file():
|
|
log.die(f'can\'t init: no {temp_manifest_filename} found in '
|
|
f'{tempdir}\n'
|
|
f' Hint: check --manifest-url={manifest_url} and '
|
|
f'--manifest-rev={manifest_rev}\n'
|
|
f' You may need to remove {west_dir} before retrying.')
|
|
|
|
# Parse the manifest to get the manifest path, if it declares one.
|
|
# Otherwise, use the URL. Ignore imports -- all we really
|
|
# want to know is if there's a "self: path:" or not.
|
|
projects = Manifest.from_file(temp_manifest,
|
|
import_flags=ImportFlag.IGNORE,
|
|
topdir=topdir).projects
|
|
manifest_project = projects[MANIFEST_PROJECT_INDEX]
|
|
if manifest_project.path:
|
|
manifest_path = manifest_project.path
|
|
else:
|
|
# We use PurePath() here in case manifest_url is a
|
|
# windows-style path. That does the right thing in that
|
|
# case, without affecting POSIX platforms, where PurePath
|
|
# is PurePosixPath.
|
|
manifest_path = PurePath(urlparse(manifest_url).path).name
|
|
|
|
manifest_abspath = topdir / manifest_path
|
|
|
|
log.dbg('moving', tempdir, 'to', manifest_abspath,
|
|
level=log.VERBOSE_EXTREME)
|
|
manifest_abspath.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
shutil.move(os.fspath(tempdir), os.fspath(manifest_abspath))
|
|
except shutil.Error as e:
|
|
log.die(e)
|
|
log.small_banner('setting manifest.path to', manifest_path)
|
|
update_config('manifest', 'path', manifest_path, topdir=topdir)
|
|
update_config('manifest', 'file', temp_manifest_filename,
|
|
topdir=topdir)
|
|
|
|
return topdir
|
|
|
|
def create(self, directory: Path, exist_ok: bool = True) -> None:
|
|
try:
|
|
directory.mkdir(parents=True, exist_ok=exist_ok)
|
|
except PermissionError:
|
|
log.die(f'Cannot initialize in {directory}: permission denied')
|
|
except FileExistsError:
|
|
log.die(f'Cannot initialize in {directory}: it already exists')
|
|
except Exception as e:
|
|
log.die(f"Can't create {directory}: {e}")
|
|
|
|
def get_head_branch(self, url: str) -> str:
|
|
# Get the branch which url's HEAD points to. Errors out if it
|
|
# can't, prints a banner if it can.
|
|
|
|
if self.git_version_info < (2, 8, 0):
|
|
# This recipe requires git 2.8.0 or later. Fall back
|
|
# if we're running on something that's too old.
|
|
return 'master'
|
|
|
|
err_msg = (f'failed getting the default branch from {url}; '
|
|
'please provide the --manifest-rev option')
|
|
|
|
# The '--quiet' option disables printing the URL to stderr.
|
|
try:
|
|
output = self.check_output(
|
|
('git', 'ls-remote', '--quiet', '--symref', url, 'HEAD')
|
|
).decode('utf-8')
|
|
except subprocess.CalledProcessError:
|
|
log.die(err_msg)
|
|
|
|
for line in output.splitlines():
|
|
if not line.startswith('ref: '):
|
|
continue
|
|
# The output looks like this:
|
|
#
|
|
# ref: refs/heads/foo HEAD
|
|
# 6145ab537fcb3adc3ee77db5f5f95e661f1e91e6 HEAD
|
|
#
|
|
# So this is the 'ref: ...' case.
|
|
#
|
|
# Per git-check-ref-format(1), references can't have tabs
|
|
# in them, so this doesn't have any weird edge cases.
|
|
without_ref = line[len('ref: '):]
|
|
if not without_ref:
|
|
continue
|
|
ret = without_ref.split('\t')[0]
|
|
log.small_banner('no --manifest-rev was given; '
|
|
f"using remote's default branch: {ret}")
|
|
return ret
|
|
|
|
log.die(err_msg)
|
|
|
|
def clone_manifest(self, url: str, rev: str, dest: str,
|
|
exist_ok=False) -> None:
|
|
log.small_banner(f'Cloning manifest repository from {url}, rev. {rev}')
|
|
if not exist_ok and exists(dest):
|
|
log.die(f'refusing to clone into existing location {dest}')
|
|
|
|
self.check_call(('git', 'init', dest))
|
|
self.check_call(('git', 'remote', 'add', 'origin', '--', url),
|
|
cwd=dest)
|
|
maybe_sha = _maybe_sha(rev)
|
|
if maybe_sha:
|
|
# Fetch the ref-space and hope the SHA is contained in
|
|
# that ref-space
|
|
self.check_call(('git', 'fetch', 'origin', '--tags',
|
|
'--', 'refs/heads/*:refs/remotes/origin/*'),
|
|
cwd=dest)
|
|
else:
|
|
# Fetch the ref-space similar to git clone plus the ref
|
|
# given by user. Redundancy is ok, for example if the user
|
|
# specifies 'heads/master'. This allows users to specify:
|
|
# pull/<no>/head for pull requests
|
|
self.check_call(('git', 'fetch', 'origin', '--tags', '--',
|
|
rev, 'refs/heads/*:refs/remotes/origin/*'),
|
|
cwd=dest)
|
|
|
|
try:
|
|
# Using show-ref to determine if rev is available in local repo.
|
|
self.check_call(('git', 'show-ref', '--', rev), cwd=dest)
|
|
local_rev = True
|
|
except subprocess.CalledProcessError:
|
|
local_rev = False
|
|
|
|
if local_rev or maybe_sha:
|
|
self.check_call(('git', 'checkout', rev), cwd=dest)
|
|
else:
|
|
self.check_call(('git', 'checkout', 'FETCH_HEAD'), cwd=dest)
|
|
|
|
class List(_ProjectCommand):
|
|
def __init__(self):
|
|
super().__init__(
|
|
'list',
|
|
'print information about projects',
|
|
textwrap.dedent('''\
|
|
Print information about projects in the west manifest,
|
|
using format strings.'''))
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
default_fmt = '{name:12} {path:28} {revision:40} {url}'
|
|
parser = self._parser(
|
|
parser_adder,
|
|
epilog=f'''\
|
|
{ACTIVE_PROJECTS_HELP}
|
|
|
|
FORMAT STRINGS
|
|
--------------
|
|
|
|
Projects are listed using a Python 3 format string. Arguments
|
|
to the format string are accessed by name.
|
|
|
|
The default format string is:
|
|
|
|
"{default_fmt}"
|
|
|
|
The following arguments are available:
|
|
|
|
- name: project name in the manifest
|
|
- url: full remote URL as specified by the manifest
|
|
- path: the relative path to the project from the top level,
|
|
as specified in the manifest where applicable
|
|
- abspath: absolute and normalized path to the project
|
|
- posixpath: like abspath, but in posix style, that is, with '/'
|
|
as the separator character instead of '\\'
|
|
- revision: project's revision as it appears in the manifest
|
|
- sha: project's revision as a SHA. Note that use of this requires
|
|
that the project has been cloned.
|
|
- cloned: "cloned" if the project has been cloned, "not-cloned"
|
|
otherwise
|
|
- clone_depth: project clone depth if specified, "None" otherwise
|
|
- groups: project groups, as a comma-separated list
|
|
''')
|
|
parser.add_argument('-a', '--all', action='store_true',
|
|
help='include inactive projects'),
|
|
parser.add_argument('--manifest-path-from-yaml', action='store_true',
|
|
help='''print the manifest repository's path
|
|
according to the manifest file YAML, which may
|
|
disagree with the manifest.path configuration
|
|
option'''),
|
|
parser.add_argument('-f', '--format', default=default_fmt,
|
|
help='''format string to use to list each
|
|
project; see FORMAT STRINGS below.''')
|
|
|
|
parser.add_argument('projects', metavar='PROJECT', nargs='*',
|
|
help='''projects (by name or path) to operate on;
|
|
see ACTIVE PROJECTS below''')
|
|
|
|
return parser
|
|
|
|
def do_run(self, args, user_args):
|
|
def sha_thunk(project):
|
|
self.die_if_no_git()
|
|
|
|
if not project.is_cloned():
|
|
log.die(f'cannot get sha for uncloned project {project.name}; '
|
|
f'run "west update {project.name}" and retry')
|
|
elif project.revision:
|
|
return project.sha(MANIFEST_REV)
|
|
else:
|
|
return f'{"N/A":40}'
|
|
|
|
def cloned_thunk(project):
|
|
self.die_if_no_git()
|
|
|
|
return "cloned" if project.is_cloned() else "not-cloned"
|
|
|
|
def delay(func, project):
|
|
return DelayFormat(partial(func, project))
|
|
|
|
self._setup_logging(args)
|
|
|
|
for project in self._projects(args.projects):
|
|
# Skip inactive projects unless the user said
|
|
# --all or named some projects explicitly.
|
|
if not (args.all or args.projects or
|
|
self.manifest.is_active(project)):
|
|
log.dbg(f'{project.name}: skipping inactive project')
|
|
continue
|
|
|
|
# Spelling out the format keys explicitly here gives us
|
|
# future-proofing if the internal Project representation
|
|
# ever changes.
|
|
#
|
|
# Using DelayFormat delays computing derived values, such
|
|
# as SHAs, unless they are specifically requested, and then
|
|
# ensures they are only computed once.
|
|
try:
|
|
if (isinstance(project, ManifestProject) and not
|
|
args.manifest_path_from_yaml):
|
|
# Special-case the manifest repository while it's
|
|
# still showing up in the 'projects' list. Yet
|
|
# more evidence we should tackle #327.
|
|
path = config.get('manifest', 'path')
|
|
apath = abspath(os.path.join(self.topdir, path))
|
|
ppath = Path(apath).as_posix()
|
|
else:
|
|
path = project.path
|
|
apath = project.abspath
|
|
ppath = project.posixpath
|
|
|
|
result = args.format.format(
|
|
name=project.name,
|
|
url=project.url or 'N/A',
|
|
path=path,
|
|
abspath=apath,
|
|
posixpath=ppath,
|
|
revision=project.revision or 'N/A',
|
|
clone_depth=project.clone_depth or "None",
|
|
cloned=delay(cloned_thunk, project),
|
|
sha=delay(sha_thunk, project),
|
|
groups=','.join(project.groups))
|
|
except KeyError as e:
|
|
# The raised KeyError seems to just put the first
|
|
# invalid argument in the args tuple, regardless of
|
|
# how many unrecognizable keys there were.
|
|
log.die(f'unknown key "{e.args[0]}" in format string '
|
|
f'{shlex.quote(args.format)}')
|
|
except IndexError:
|
|
self.parser.print_usage()
|
|
log.die(f'invalid format string {shlex.quote(args.format)}')
|
|
except subprocess.CalledProcessError:
|
|
log.die(f'subprocess failed while listing {project.name}')
|
|
|
|
log.inf(result, colorize=False)
|
|
|
|
class ManifestCommand(_ProjectCommand):
|
|
# The slightly weird naming is to avoid a conflict with
|
|
# west.manifest.Manifest.
|
|
|
|
def __init__(self):
|
|
super(ManifestCommand, self).__init__(
|
|
'manifest',
|
|
'manage the west manifest',
|
|
textwrap.dedent('''\
|
|
Manages the west manifest.
|
|
|
|
The following actions are available. You must give exactly one.
|
|
|
|
- --resolve: print the current manifest with all imports applied,
|
|
as an equivalent single manifest file. Any imported manifests
|
|
must be cloned locally (with "west update").
|
|
|
|
- --freeze: like --resolve, but with all project revisions
|
|
converted to their current SHAs, based on the latest manifest-rev
|
|
branches. All projects must be cloned (with "west update").
|
|
|
|
- --validate: print an error and exit the process unsuccessfully
|
|
if the current manifest cannot be successfully parsed.
|
|
If the manifest can be parsed, print nothing and exit
|
|
successfully.
|
|
|
|
- --path: print the path to the top level manifest file.
|
|
If this file uses imports, it will not contain all the
|
|
manifest data.
|
|
|
|
If the manifest file does not use imports, and all project
|
|
revisions are SHAs, the --freeze and --resolve output will
|
|
be identical after a "west update".
|
|
'''),
|
|
accepts_unknown_args=False)
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
parser = self._parser(parser_adder)
|
|
|
|
group = parser.add_mutually_exclusive_group(required=True)
|
|
group.add_argument('--resolve', action='store_true',
|
|
help='print the manifest with all imports resolved')
|
|
group.add_argument('--freeze', action='store_true',
|
|
help='''print the resolved manifest with SHAs for
|
|
all project revisions''')
|
|
group.add_argument('--validate', action='store_true',
|
|
help='''validate the current manifest,
|
|
exiting with an error if there are issues''')
|
|
group.add_argument('--path', action='store_true',
|
|
help="print the top level manifest file's path")
|
|
|
|
group = parser.add_argument_group('options for --resolve and --freeze')
|
|
group.add_argument('-o', '--out',
|
|
help='output file, default is standard output')
|
|
|
|
return parser
|
|
|
|
def do_run(self, args, user_args):
|
|
self._setup_logging(args)
|
|
|
|
# Since the user is explicitly managing the manifest, we are
|
|
# deliberately loading it again instead of using self.manifest
|
|
# to emit debug logs if enabled, which are turned off when the
|
|
# manifest is initially parsed in main.py.
|
|
#
|
|
# The code in main.py is usually responsible for handling any
|
|
# errors and printing useful messages. We re-do error checking
|
|
# for manifest-related errors that it won't handle.
|
|
try:
|
|
manifest = Manifest.from_file(topdir=self.topdir)
|
|
except _ManifestImportDepth:
|
|
log.die("cannot resolve manifest -- is there a loop?")
|
|
except ManifestImportFailed as mif:
|
|
log.die(f"manifest import failed\n Project: {mif.project}\n "
|
|
f"File: {mif.filename}")
|
|
except (MalformedManifest, ManifestVersionError) as e:
|
|
log.die('\n '.join(str(arg) for arg in e.args))
|
|
dump_kwargs = {'default_flow_style': False,
|
|
'sort_keys': False}
|
|
|
|
if args.validate:
|
|
pass # nothing more to do
|
|
elif args.resolve:
|
|
self._dump(args, manifest.as_yaml(**dump_kwargs))
|
|
elif args.freeze:
|
|
self._dump(args, manifest.as_frozen_yaml(**dump_kwargs))
|
|
elif args.path:
|
|
log.inf(manifest.path)
|
|
else:
|
|
# Can't happen.
|
|
raise RuntimeError(f'internal error: unhandled args {args}')
|
|
|
|
def _dump(self, args, to_dump):
|
|
if args.out:
|
|
with open(args.out, 'w') as f:
|
|
f.write(to_dump)
|
|
else:
|
|
sys.stdout.write(to_dump)
|
|
|
|
class Diff(_ProjectCommand):
|
|
def __init__(self):
|
|
super().__init__(
|
|
'diff',
|
|
'"git diff" for one or more projects',
|
|
'Runs "git diff" on each of the specified projects.')
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
parser = self._parser(parser_adder,
|
|
epilog=ACTIVE_CLONED_PROJECTS_HELP)
|
|
parser.add_argument('projects', metavar='PROJECT', nargs='*',
|
|
help='''projects (by name or path) to operate on;
|
|
defaults to active cloned projects''')
|
|
parser.add_argument('-a', '--all', action='store_true',
|
|
help='include output for inactive projects')
|
|
return parser
|
|
|
|
def do_run(self, args, ignored):
|
|
self.die_if_no_git()
|
|
self._setup_logging(args)
|
|
|
|
failed = []
|
|
no_diff = 0
|
|
# We may need to force git to use colors if the user wants them,
|
|
# which it won't do ordinarily since stdout is not a terminal.
|
|
color = ['--color=always'] if log.use_color() else []
|
|
for project in self._cloned_projects(args, only_active=not args.all):
|
|
# Use paths that are relative to the base directory to make it
|
|
# easier to see where the changes are
|
|
cp = project.git(['diff', f'--src-prefix={project.path}/',
|
|
f'--dst-prefix={project.path}/',
|
|
'--exit-code'] + color,
|
|
capture_stdout=True, capture_stderr=True,
|
|
check=False)
|
|
if cp.returncode == 0:
|
|
no_diff += 1
|
|
if cp.returncode == 1 or log.VERBOSE > log.VERBOSE_NONE:
|
|
log.banner(f'diff for {project.name_and_path}:')
|
|
log.inf(cp.stdout.decode('utf-8'))
|
|
elif cp.returncode:
|
|
failed.append(project)
|
|
if failed:
|
|
self._handle_failed(args, failed)
|
|
elif log.VERBOSE <= log.VERBOSE_NONE:
|
|
log.inf(f"Empty diff in {no_diff} projects.")
|
|
|
|
class Status(_ProjectCommand):
|
|
def __init__(self):
|
|
super().__init__(
|
|
'status',
|
|
'"git status" for one or more projects',
|
|
"Runs 'git status' for each of the specified projects.")
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
parser = self._parser(parser_adder,
|
|
epilog=ACTIVE_CLONED_PROJECTS_HELP)
|
|
parser.add_argument('projects', metavar='PROJECT', nargs='*',
|
|
help='''projects (by name or path) to operate on;
|
|
defaults to active cloned projects''')
|
|
parser.add_argument('-a', '--all', action='store_true',
|
|
help='include output for inactive projects')
|
|
return parser
|
|
|
|
def do_run(self, args, user_args):
|
|
self.die_if_no_git()
|
|
self._setup_logging(args)
|
|
|
|
failed = []
|
|
for project in self._cloned_projects(args, only_active=not args.all):
|
|
log.banner(f'status of {project.name_and_path}:')
|
|
try:
|
|
project.git('status', extra_args=user_args)
|
|
except subprocess.CalledProcessError:
|
|
failed.append(project)
|
|
self._handle_failed(args, failed)
|
|
|
|
class Update(_ProjectCommand):
|
|
|
|
def __init__(self):
|
|
super().__init__(
|
|
'update',
|
|
'update projects described in west manifest',
|
|
textwrap.dedent('''\
|
|
Updates active projects defined in the manifest file as follows:
|
|
|
|
1. Clone the project if necessary
|
|
2. If necessary, fetch the project's revision from its remote
|
|
(see "fetching behavior" below)
|
|
3. Reset the manifest-rev branch to the current manifest revision
|
|
4. Check out the new manifest-rev commit as a detached
|
|
HEAD (the default), or keep/rebase existing checked out branches
|
|
(see "checked out branch behavior")
|
|
|
|
You must have already created a west workspace with "west init".
|
|
|
|
This command does not alter the manifest repository's contents.''')
|
|
)
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
parser = self._parser(parser_adder)
|
|
|
|
parser.add_argument('--stats', action='store_true',
|
|
help='''print performance statistics for
|
|
update operations''')
|
|
|
|
group = parser.add_argument_group(
|
|
title='local project clone caches',
|
|
description=textwrap.dedent('''\
|
|
Projects are usually initialized by fetching from their URLs, but
|
|
they can also be cloned from caches on the local file system.'''))
|
|
group.add_argument('--name-cache',
|
|
help='''cached repositories are in subdirectories
|
|
matching the names of projects to update''')
|
|
group.add_argument('--path-cache',
|
|
help='''cached repositories are in the same relative
|
|
paths as the workspace being updated''')
|
|
|
|
group = parser.add_argument_group(
|
|
title='fetching behavior',
|
|
description='By default, west update tries to avoid fetching.')
|
|
group.add_argument('-f', '--fetch', dest='fetch_strategy',
|
|
choices=['always', 'smart'],
|
|
help='''how to fetch projects when updating:
|
|
"always" fetches every project before update,
|
|
while "smart" (default) skips fetching projects
|
|
whose revisions are SHAs or tags available
|
|
locally''')
|
|
group.add_argument('-o', '--fetch-opt', action='append', default=[],
|
|
help='''additional option to pass to 'git fetch'
|
|
if fetching is necessary (e.g. 'o=--depth=1');
|
|
may be given more than once''')
|
|
group.add_argument('-n', '--narrow', action='store_true',
|
|
help='''fetch just the project revision if fetching
|
|
is necessary; skip fetching tags (may not work for
|
|
SHA revisions depending on the Git host)''')
|
|
|
|
group = parser.add_argument_group(
|
|
title='checked out branch behavior',
|
|
description=textwrap.dedent('''\
|
|
By default, locally checked out branches are left behind
|
|
when manifest-rev commits are checked out.'''))
|
|
group.add_argument('-k', '--keep-descendants', action='store_true',
|
|
help='''if a checked out branch is a descendant
|
|
of the new manifest-rev, leave it checked out
|
|
instead (takes priority over --rebase)''')
|
|
group.add_argument('-r', '--rebase', action='store_true',
|
|
help='''rebase any checked out branch onto the new
|
|
manifest-rev instead (leaving behind partial
|
|
rebases on error)''')
|
|
|
|
group = parser.add_argument_group(
|
|
title='advanced options')
|
|
group.add_argument('--group-filter', '--gf', action='append',
|
|
default=[], metavar='FILTER', dest='group_filter',
|
|
help='''proceed as if FILTER was appended to
|
|
manifest.group-filter; may be given multiple
|
|
times''')
|
|
|
|
group = parser.add_argument_group('deprecated options')
|
|
group.add_argument('-x', '--exclude-west', action='store_true',
|
|
help='ignored for backwards compatibility')
|
|
|
|
parser.add_argument('projects', metavar='PROJECT', nargs='*',
|
|
help='''projects (by name or path) to operate on;
|
|
defaults to all active projects''')
|
|
|
|
return parser
|
|
|
|
def do_run(self, args, _):
|
|
self.die_if_no_git()
|
|
self._setup_logging(args)
|
|
self.init_state(args)
|
|
|
|
# We can't blindly call self._projects() here: manifests with
|
|
# imports are limited to plain 'west update', and cannot use
|
|
# 'west update PROJECT [...]'.
|
|
if not self.args.projects:
|
|
self.update_all()
|
|
else:
|
|
self.update_some()
|
|
|
|
def init_state(self, args):
|
|
# Helper for initializing instance state in response to
|
|
# command line args and configuration files.
|
|
|
|
self.args = args
|
|
if args.exclude_west:
|
|
log.wrn('ignoring --exclude-west')
|
|
|
|
self.narrow = args.narrow or config.getboolean('update', 'narrow',
|
|
fallback=False)
|
|
self.path_cache = args.path_cache or config.get('update', 'path-cache',
|
|
fallback=None)
|
|
self.name_cache = args.name_cache or config.get('update', 'name-cache',
|
|
fallback=None)
|
|
self.sync_submodules = config.getboolean('update', 'sync-submodules',
|
|
fallback=True)
|
|
|
|
self.group_filter: List[str] = []
|
|
|
|
def handle(group_filter_item):
|
|
item = group_filter_item.strip()
|
|
if not item.startswith(('-', '+')):
|
|
log.die(f'invalid --group-filter item {item}: '
|
|
'must start with - or +')
|
|
if not is_project_group(item[1:]):
|
|
log.die(f'invalid --group-filter item {item}: '
|
|
f'"{item[1:]}" is not a valid group name')
|
|
self.group_filter.append(item)
|
|
|
|
for item in args.group_filter:
|
|
if ',' in item:
|
|
for split_item in item.split(','):
|
|
handle(split_item)
|
|
else:
|
|
handle(item)
|
|
|
|
self.fs = self.fetch_strategy()
|
|
|
|
def update_all(self):
|
|
# Plain 'west update' is the 'easy' case: since the user just
|
|
# wants us to update everything, we don't have to keep track
|
|
# of projects appearing or disappearing as a result of fetching
|
|
# new revisions from projects with imports.
|
|
#
|
|
# So we just re-parse the manifest, but force west.manifest to
|
|
# call our importer whenever it encounters an import statement
|
|
# in a project, allowing us to control the recursion so it
|
|
# always uses the latest manifest data.
|
|
self.updated = set()
|
|
|
|
self.manifest = Manifest.from_file(
|
|
importer=self.update_importer,
|
|
import_flags=ImportFlag.FORCE_PROJECTS)
|
|
|
|
failed = []
|
|
for project in self.manifest.projects:
|
|
if (isinstance(project, ManifestProject) or
|
|
project.name in self.updated):
|
|
continue
|
|
try:
|
|
if not self.project_is_active(project):
|
|
log.dbg(f'{project.name}: skipping inactive project')
|
|
continue
|
|
self.update(project)
|
|
self.updated.add(project.name)
|
|
except subprocess.CalledProcessError:
|
|
failed.append(project)
|
|
self._handle_failed(self.args, failed)
|
|
|
|
def update_importer(self, project, path):
|
|
if isinstance(project, ManifestProject):
|
|
if not project.is_cloned():
|
|
log.die("manifest repository {project.abspath} was deleted")
|
|
else:
|
|
# There's no need to call self.project_is_active(),
|
|
# because the Manifest API guarantees that 'groups' cannot
|
|
# be combined with 'import' within a single project.
|
|
#
|
|
# That's good, because the semantics would be kind of hard
|
|
# to specify in this case.
|
|
assert not project.groups
|
|
|
|
self.update(project)
|
|
self.updated.add(project.name)
|
|
|
|
try:
|
|
return _manifest_content_at(project, path)
|
|
except FileNotFoundError:
|
|
# FIXME we need each project to have back-pointers
|
|
# to the manifest file where it was defined, so we can
|
|
# tell the user better context than just "run -vvv", which
|
|
# is a total fire hose.
|
|
name = project.name
|
|
sha = project.sha(QUAL_MANIFEST_REV)
|
|
if log.VERBOSE < log.VERBOSE_EXTREME:
|
|
suggest_vvv = ('\n'
|
|
' Use "west -vvv update" to debug.')
|
|
else:
|
|
suggest_vvv = ''
|
|
log.die(f"can't import from project {name}\n"
|
|
f' Expected to import from {path} at revision {sha}\n'
|
|
f' Hint: possible manifest file fixes for {name}:\n'
|
|
f' - set "revision:" to a git ref with this file '
|
|
f'at URL {project.url}\n'
|
|
' - remove the "import:"' + suggest_vvv)
|
|
|
|
def update_some(self):
|
|
# The 'west update PROJECT [...]' style invocation is only
|
|
# implemented for projects defined within the manifest
|
|
# repository.
|
|
#
|
|
# It's unclear how to do this properly in the case of
|
|
# a project A whose definition is imported from
|
|
# another project B, especially when B.revision is not
|
|
# a fixed SHA. Do we forcibly need to update B first?
|
|
# Should we skip it? Should it be configurable? Etc.
|
|
#
|
|
# For now, just refuse to do so. We can try to relax
|
|
# this restriction if it proves cumbersome.
|
|
|
|
if not self.has_manifest or self.manifest.has_imports:
|
|
projects = self.toplevel_projects()
|
|
assert self.has_manifest # toplevel_projects() must ensure this.
|
|
else:
|
|
projects = self._projects(self.args.projects)
|
|
|
|
failed = []
|
|
for project in projects:
|
|
if isinstance(project, ManifestProject):
|
|
continue
|
|
try:
|
|
self.update(project)
|
|
except subprocess.CalledProcessError:
|
|
failed.append(project)
|
|
self._handle_failed(self.args, failed)
|
|
|
|
def toplevel_projects(self):
|
|
# Return a list of projects from self.args.projects, or scream
|
|
# and die if any projects are either unknown or not defined in
|
|
# the manifest repository.
|
|
#
|
|
# As a side effect, ensures self.manifest is set.
|
|
|
|
ids = self.args.projects
|
|
assert ids
|
|
|
|
self.manifest = Manifest.from_file(
|
|
import_flags=ImportFlag.IGNORE_PROJECTS)
|
|
mr_projects, mr_unknown = projects_unknown(self.manifest, ids)
|
|
if not mr_unknown:
|
|
return mr_projects
|
|
|
|
try:
|
|
self.manifest = Manifest.from_file()
|
|
except ManifestImportFailed:
|
|
log.die('one or more projects are unknown or defined via '
|
|
'imports; please run plain "west update".')
|
|
|
|
_, unknown = projects_unknown(self.manifest, ids)
|
|
if unknown:
|
|
die_unknown(unknown)
|
|
else:
|
|
# All of the ids are known projects, but some of them
|
|
# are not defined in the manifest repository.
|
|
mr_unknown_set = set(mr_unknown)
|
|
from_projects = [p for p in ids if p in mr_unknown_set]
|
|
log.die('refusing to update project: ' +
|
|
" ".join(from_projects) + '\n' +
|
|
' It or they were resolved via project imports.\n'
|
|
' Only plain "west update" can currently update them.')
|
|
|
|
def fetch_strategy(self):
|
|
cfg = config.get('update', 'fetch', fallback=None)
|
|
if cfg is not None and cfg not in ('always', 'smart'):
|
|
log.wrn(f'ignoring invalid config update.fetch={cfg}; '
|
|
'choices: always, smart')
|
|
cfg = None
|
|
if self.args.fetch_strategy:
|
|
return self.args.fetch_strategy
|
|
elif cfg:
|
|
return cfg
|
|
else:
|
|
return 'smart'
|
|
|
|
def update_submodules(self, project):
|
|
# Updates given project submodules by using
|
|
# 'git submodule update --init --checkout --recursive' command
|
|
# from the project.path location.
|
|
if not project.submodules:
|
|
return
|
|
|
|
submodules = project.submodules
|
|
submodules_update_strategy = ('--rebase' if self.args.rebase
|
|
else '--checkout')
|
|
# For the list type, update given list of submodules.
|
|
if isinstance(submodules, list):
|
|
for submodule in submodules:
|
|
if self.sync_submodules:
|
|
project.git(['submodule', 'sync', '--recursive',
|
|
'--', submodule.path])
|
|
project.git(['submodule', 'update',
|
|
'--init', submodules_update_strategy,
|
|
'--recursive', submodule.path])
|
|
# For the bool type, update all project submodules
|
|
elif isinstance(submodules, bool):
|
|
if self.sync_submodules:
|
|
project.git(['submodule', 'sync', '--recursive'])
|
|
project.git(['submodule', 'update', '--init',
|
|
submodules_update_strategy, '--recursive'])
|
|
|
|
def update(self, project):
|
|
if self.args.stats:
|
|
stats = dict()
|
|
update_start = perf_counter()
|
|
else:
|
|
stats = None
|
|
take_stats = stats is not None
|
|
|
|
log.banner(f'updating {project.name_and_path}:')
|
|
|
|
# Make sure we've got a project to work with.
|
|
self.ensure_cloned(project, stats, take_stats)
|
|
|
|
# Point refs/heads/manifest-rev at project.revision,
|
|
# fetching it from the remote if necessary.
|
|
self.set_new_manifest_rev(project, stats, take_stats)
|
|
|
|
# Clean up refs/west/*. At some point, we should only do this
|
|
# if we've fetched, but we're leaving it here to clean up
|
|
# garbage in people's repositories introduced by previous
|
|
# versions of west that left refs in place here.
|
|
self.clean_refs_west(project, stats, take_stats)
|
|
|
|
# Make sure HEAD is pointing at *something*.
|
|
self.ensure_head_ok(project, stats, take_stats)
|
|
|
|
# Convert manifest-rev to a SHA.
|
|
sha = self.manifest_rev_sha(project, stats, take_stats)
|
|
|
|
# Based on the new manifest-rev SHA, HEAD, and the --rebase
|
|
# and --keep-descendants options, decide what we need to do
|
|
# now.
|
|
current_branch, is_ancestor, try_rebase = self.decide_update_strategy(
|
|
project, sha, stats, take_stats)
|
|
|
|
# Finish the update. This may be a nop if we're keeping
|
|
# descendants.
|
|
if self.args.keep_descendants and is_ancestor:
|
|
# A descendant is currently checked out and keep_descendants was
|
|
# given, so there's nothing more to do.
|
|
log.inf(f'west update: left descendant branch '
|
|
f'"{current_branch}" checked out; current status:')
|
|
if take_stats:
|
|
start = perf_counter()
|
|
project.git('status')
|
|
if take_stats:
|
|
stats['get current status'] = perf_counter - start
|
|
elif try_rebase:
|
|
# Attempt a rebase.
|
|
log.inf(f'west update: rebasing to {MANIFEST_REV} {sha}')
|
|
if take_stats:
|
|
start = perf_counter()
|
|
project.git('rebase ' + QUAL_MANIFEST_REV)
|
|
if take_stats:
|
|
stats['rebase onto new manifest-rev'] = perf_counter() - start
|
|
else:
|
|
# We can't keep a descendant or rebase, so just check
|
|
# out the new detached HEAD, then print some helpful context.
|
|
if take_stats:
|
|
start = perf_counter()
|
|
project.git('checkout --detach ' + sha)
|
|
if take_stats:
|
|
stats['checkout new manifest-rev'] = perf_counter() - start
|
|
_post_checkout_help(project, current_branch, sha, is_ancestor)
|
|
|
|
# Update project submodules, if it has any.
|
|
if take_stats:
|
|
start = perf_counter()
|
|
self.update_submodules(project)
|
|
if take_stats:
|
|
stats['update submodules'] = perf_counter() - start
|
|
|
|
# Print performance statistics.
|
|
if take_stats:
|
|
update_total = perf_counter() - update_start
|
|
slop = update_total - sum(stats.values())
|
|
stats['other work'] = slop
|
|
stats['TOTAL'] = update_total
|
|
log.inf('performance statistics:')
|
|
for stat, value in stats.items():
|
|
log.inf(f' {stat}: {value} sec')
|
|
|
|
def ensure_cloned(self, project, stats, take_stats):
|
|
# update() helper. Make sure project is cloned and initialized.
|
|
|
|
if take_stats:
|
|
start = perf_counter()
|
|
cloned = project.is_cloned()
|
|
if take_stats:
|
|
stats['check if cloned'] = perf_counter() - start
|
|
if not cloned:
|
|
if take_stats:
|
|
start = perf_counter()
|
|
self.init_project(project)
|
|
if take_stats:
|
|
stats['init'] = perf_counter() - start
|
|
|
|
def init_project(self, project):
|
|
# update() helper. Initialize an uncloned project repository.
|
|
# If there's a local clone available, it uses that. Otherwise,
|
|
# it just creates the local repository and sets up the
|
|
# convenience remote without fetching anything from the network.
|
|
|
|
cache_dir = self.project_cache(project)
|
|
|
|
if cache_dir is None:
|
|
log.small_banner(f'{project.name}: initializing')
|
|
|
|
init_cmd = ['init', project.abspath]
|
|
# Silence the very verbose and repetitive init.defaultBranch
|
|
# warning (10 lines per new git clone). The branch
|
|
# 'placeholder' will never have any commit so it will never
|
|
# actually exist.
|
|
if self.git_version_info >= (2, 28, 0):
|
|
init_cmd.insert(1, '--initial-branch=init_placeholder')
|
|
|
|
project.git(init_cmd, cwd=self.topdir)
|
|
|
|
# This remote is added as a convenience for the user.
|
|
# However, west always fetches project data by URL, not name.
|
|
# The user is therefore free to change the URL of this remote.
|
|
project.git(f'remote add -- {project.remote_name} {project.url}')
|
|
else:
|
|
log.small_banner(f'{project.name}: cloning from {cache_dir}')
|
|
# Clone the project from a local cache repository. Set the
|
|
# remote name to the value that would be used without a
|
|
# cache.
|
|
project.git(['clone', '--origin', project.remote_name,
|
|
cache_dir, project.abspath], cwd=self.topdir)
|
|
# Reset the remote's URL to the project's fetch URL.
|
|
project.git(['remote', 'set-url', project.remote_name,
|
|
project.url])
|
|
# Make sure we have a detached HEAD so we can delete the
|
|
# local branch created by git clone.
|
|
project.git('checkout --quiet --detach HEAD')
|
|
# Find the name of any local branch created by git clone.
|
|
# West commits to only touching 'manifest-rev' in the
|
|
# local branch name space.
|
|
local_branches = project.git(
|
|
['for-each-ref', '--format', '%(refname)', 'refs/heads/*'],
|
|
capture_stdout=True).stdout.decode('utf-8').splitlines()
|
|
# This should contain at most one branch in current
|
|
# versions of git, but we might as well get them all just
|
|
# in case that changes.
|
|
for branch in local_branches:
|
|
if not branch:
|
|
continue
|
|
# This is safe: it can't be garbage collected by git before we
|
|
# have a chance to use it, because we have another ref, namely
|
|
# f'refs/remotes/{project.remote_name}/{branch}'.
|
|
project.git(['update-ref', '-d', branch])
|
|
|
|
def project_cache(self, project):
|
|
# Find the absolute path to a pre-existing local clone of a project
|
|
# and return it. If the search fails, return None.
|
|
|
|
if self.name_cache is not None:
|
|
maybe = Path(self.name_cache) / project.name
|
|
if maybe.is_dir():
|
|
log.dbg(
|
|
f'found {project.name} in --name-cache {self.name_cache}',
|
|
level=log.VERBOSE_VERY)
|
|
return os.fspath(maybe)
|
|
else:
|
|
log.dbg(
|
|
f'{project.name} not in --name-cache {self.name_cache}',
|
|
level=log.VERBOSE_VERY)
|
|
elif self.path_cache is not None:
|
|
maybe = Path(self.path_cache) / project.path
|
|
if maybe.is_dir():
|
|
log.dbg(
|
|
f'found {project.path} in --path-cache {self.path_cache}',
|
|
level=log.VERBOSE_VERY)
|
|
return os.fspath(maybe)
|
|
else:
|
|
log.dbg(
|
|
f'{project.path} not in --path-cache {self.path_cache}',
|
|
level=log.VERBOSE_VERY)
|
|
|
|
return None
|
|
|
|
def set_new_manifest_rev(self, project, stats, take_stats):
|
|
# update() helper. Make sure project's manifest-rev is set to
|
|
# the latest value it should be.
|
|
|
|
if self.fs == 'always' or _rev_type(project) not in ('tag', 'commit'):
|
|
self.fetch(project, stats, take_stats)
|
|
else:
|
|
log.dbg('skipping unnecessary fetch')
|
|
if take_stats:
|
|
start = perf_counter()
|
|
_update_manifest_rev(project, f'{project.revision}^{{commit}}')
|
|
if take_stats:
|
|
stats['set manifest-rev'] = perf_counter() - start
|
|
|
|
def fetch(self, project, stats, take_stats):
|
|
# Fetches rev (or project.revision) from project.url in a way that
|
|
# guarantees any branch, tag, or SHA (that's reachable from a
|
|
# branch or a tag) available on project.url is part of what got
|
|
# fetched.
|
|
#
|
|
# Returns a git revision which hopefully can be peeled to the
|
|
# newly-fetched SHA corresponding to rev. "Hopefully" because
|
|
# there are many ways to spell a revision, and they haven't all
|
|
# been extensively tested.
|
|
|
|
if take_stats:
|
|
start = perf_counter()
|
|
|
|
rev = project.revision
|
|
|
|
# Fetch the revision into the local ref space.
|
|
#
|
|
# The following two-step approach avoids a "trying to write
|
|
# non-commit object" error when the revision is an annotated
|
|
# tag. ^{commit} type peeling isn't supported for the <src> in a
|
|
# <src>:<dst> refspec, so we have to do it separately.
|
|
if _maybe_sha(rev) and not self.narrow:
|
|
# We can't in general fetch a SHA from a remote, as some hosts
|
|
# forbid it for security reasons. Let's hope it's reachable
|
|
# from some branch.
|
|
refspec = f'refs/heads/*:{QUAL_REFS}*'
|
|
next_manifest_rev = project.revision
|
|
else:
|
|
# Either the revision is definitely not a SHA and is
|
|
# therefore safe to fetch directly, or the user said
|
|
# that's OK. This avoids fetching unnecessary refs from
|
|
# the remote.
|
|
#
|
|
# We update manifest-rev to FETCH_HEAD instead of using a
|
|
# refspec in case the revision is a tag, which we can't use
|
|
# from a refspec.
|
|
refspec = project.revision
|
|
next_manifest_rev = 'FETCH_HEAD^{commit}'
|
|
|
|
log.small_banner(f'{project.name}: fetching, need revision {rev}')
|
|
# --tags is required to get tags if we're not run as 'west
|
|
# update --narrow', since the remote is specified as a URL.
|
|
tags = (['--tags'] if not self.narrow else [])
|
|
clone_depth = (['--depth', str(project.clone_depth)] if
|
|
project.clone_depth else [])
|
|
# -f is needed to avoid errors in case multiple remotes are
|
|
# present, at least one of which contains refs that can't be
|
|
# fast-forwarded to our local ref space.
|
|
project.git(['fetch', '-f'] + tags + clone_depth +
|
|
self.args.fetch_opt +
|
|
['--', project.url, refspec])
|
|
|
|
if take_stats:
|
|
stats['fetch'] = perf_counter() - start
|
|
|
|
# Update manifest-rev, leaving an entry in the reflog.
|
|
if take_stats:
|
|
start = perf_counter()
|
|
|
|
new_ref = project.sha(next_manifest_rev)
|
|
_update_manifest_rev(project, new_ref)
|
|
|
|
if take_stats:
|
|
stats['set manifest-rev'] = perf_counter() - start
|
|
|
|
@staticmethod
|
|
def clean_refs_west(project, stats, take_stats):
|
|
# update() helper. Make sure refs/west/* is empty after
|
|
# setting the new manifest-rev.
|
|
#
|
|
# Head of manifest-rev is now pointing to current manifest revision.
|
|
# Thus it is safe to unconditionally clear out the refs/west space.
|
|
#
|
|
# Doing this here instead of in Update.fetch() ensures that it
|
|
# gets cleaned up when users upgrade from older versions of
|
|
# west (like 0.6.x) that didn't handle this properly.
|
|
#
|
|
# In the future, this can be moved into Update.fetch() after
|
|
# the install base of older west versions is expected to be
|
|
# smaller.
|
|
if take_stats:
|
|
start = perf_counter()
|
|
_clean_west_refspace(project)
|
|
if take_stats:
|
|
stats['clean up refs/west/*'] = perf_counter() - start
|
|
|
|
@staticmethod
|
|
def ensure_head_ok(project, stats, take_stats):
|
|
# update() helper. Ensure HEAD points at something reasonable.
|
|
|
|
if take_stats:
|
|
start = perf_counter()
|
|
head_ok = _head_ok(project)
|
|
if take_stats:
|
|
stats['check HEAD is ok'] = perf_counter() - start
|
|
if not head_ok:
|
|
# If nothing is checked out (which usually only happens if
|
|
# we called Update.init_project() above), check out
|
|
# 'manifest-rev' in a detached HEAD state.
|
|
#
|
|
# Otherwise, the initial state would have nothing checked
|
|
# out, and HEAD would point to a non-existent
|
|
# refs/heads/master branch (that would get created if the
|
|
# user makes an initial commit). Among other things, this
|
|
# ensures the rev-parse --abbrev-ref HEAD which happens
|
|
# later in the update() will always succeed.
|
|
#
|
|
# The --detach flag is strictly redundant here, because
|
|
# the refs/heads/<branch> form already detaches HEAD, but
|
|
# it avoids a spammy detached HEAD warning from Git.
|
|
if take_stats:
|
|
start = perf_counter()
|
|
project.git('checkout --detach ' + QUAL_MANIFEST_REV)
|
|
if take_stats:
|
|
stats['checkout new manifest-rev'] = perf_counter() - start
|
|
|
|
@staticmethod
|
|
def manifest_rev_sha(project, stats, take_stats):
|
|
# update() helper. Get the SHA for manifest-rev.
|
|
|
|
try:
|
|
if take_stats:
|
|
start = perf_counter()
|
|
return project.sha(QUAL_MANIFEST_REV)
|
|
if take_stats:
|
|
stats['get new manifest-rev SHA'] = perf_counter() - start
|
|
except subprocess.CalledProcessError:
|
|
# This is a sign something's really wrong. Add more help.
|
|
log.err(f'no SHA for branch {MANIFEST_REV} '
|
|
f'in {project.name_and_path}; was the branch deleted?')
|
|
raise
|
|
|
|
def decide_update_strategy(self, project, sha, stats, take_stats):
|
|
# update() helper. Decide on whether we have an ancestor
|
|
# branch or whether we should try to rebase.
|
|
|
|
if take_stats:
|
|
start = perf_counter()
|
|
cp = project.git('rev-parse --abbrev-ref HEAD', capture_stdout=True)
|
|
if take_stats:
|
|
stats['get current branch HEAD'] = perf_counter() - start
|
|
current_branch = cp.stdout.decode('utf-8').strip()
|
|
if current_branch != 'HEAD':
|
|
if take_stats:
|
|
start = perf_counter()
|
|
is_ancestor = project.is_ancestor_of(sha, current_branch)
|
|
if take_stats:
|
|
stats['check if HEAD is ancestor of manifest-rev'] = \
|
|
perf_counter() - start
|
|
try_rebase = self.args.rebase
|
|
else: # HEAD means no branch is checked out.
|
|
# If no branch is checked out, 'rebase' and
|
|
# 'keep_descendants' don't matter.
|
|
is_ancestor = False
|
|
try_rebase = False
|
|
|
|
return current_branch, is_ancestor, try_rebase
|
|
|
|
def project_is_active(self, project):
|
|
return self.manifest.is_active(project, extra_filter=self.group_filter)
|
|
|
|
class ForAll(_ProjectCommand):
|
|
def __init__(self):
|
|
super().__init__(
|
|
'forall',
|
|
'run a command in one or more local projects',
|
|
textwrap.dedent('''\
|
|
Runs a shell (on a Unix OS) or batch (on Windows) command
|
|
within the repository of each of the specified PROJECTs.
|
|
|
|
If the command has multiple words, you must quote the -c
|
|
option to prevent the shell from splitting it up. Since
|
|
the command is run through the shell, you can use
|
|
wildcards and the like.
|
|
|
|
For example, the following command will list the contents
|
|
of proj-1's and proj-2's repositories on Linux and macOS,
|
|
in long form:
|
|
|
|
west forall -c "ls -l" proj-1 proj-2
|
|
'''))
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
parser = self._parser(parser_adder,
|
|
epilog=ACTIVE_CLONED_PROJECTS_HELP)
|
|
parser.add_argument('-c', dest='subcommand', metavar='COMMAND',
|
|
required=True)
|
|
parser.add_argument('-a', '--all', action='store_true',
|
|
help='include inactive projects'),
|
|
parser.add_argument('projects', metavar='PROJECT', nargs='*',
|
|
help='''projects (by name or path) to operate on;
|
|
defaults to active cloned projects''')
|
|
return parser
|
|
|
|
def do_run(self, args, user_args):
|
|
self._setup_logging(args)
|
|
|
|
failed = []
|
|
for project in self._cloned_projects(args, only_active=not args.all):
|
|
log.banner(
|
|
f'running "{args.subcommand}" in {project.name_and_path}:')
|
|
rc = subprocess.Popen(args.subcommand, shell=True,
|
|
cwd=project.abspath).wait()
|
|
if rc:
|
|
failed.append(project)
|
|
self._handle_failed(args, failed)
|
|
|
|
class Topdir(_ProjectCommand):
|
|
def __init__(self):
|
|
super().__init__(
|
|
'topdir',
|
|
'print the top level directory of the workspace',
|
|
textwrap.dedent('''\
|
|
Prints the absolute path of the current west workspace's
|
|
top directory.
|
|
|
|
This is the directory containing .west. All project
|
|
paths in the manifest are relative to this top directory.'''))
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
return self._parser(parser_adder)
|
|
|
|
def do_run(self, args, user_args):
|
|
log.inf(PurePath(self.topdir).as_posix())
|
|
|
|
class SelfUpdate(_ProjectCommand):
|
|
def __init__(self):
|
|
super().__init__(
|
|
'selfupdate',
|
|
'deprecated; exists for backwards compatibility',
|
|
'Do not use. You can upgrade west with pip only from v0.6.0.')
|
|
|
|
def do_add_parser(self, parser_adder):
|
|
return self._parser(parser_adder)
|
|
|
|
def do_run(self, args, user_args):
|
|
log.die(self.description)
|
|
|
|
#
|
|
# Private helper routines.
|
|
#
|
|
|
|
def _clean_west_refspace(project):
|
|
# Clean the refs/west space to ensure they do not show up in 'git log'.
|
|
|
|
# Get all the ref names that start with refs/west/.
|
|
list_refs_cmd = ('for-each-ref --format="%(refname)" -- ' +
|
|
QUAL_REFS + '**')
|
|
cp = project.git(list_refs_cmd, capture_stdout=True)
|
|
west_references = cp.stdout.decode('utf-8').strip()
|
|
|
|
# Safely delete each one.
|
|
for ref in west_references.splitlines():
|
|
delete_ref_cmd = 'update-ref -d ' + ref
|
|
project.git(delete_ref_cmd)
|
|
|
|
def _update_manifest_rev(project, new_manifest_rev):
|
|
project.git(['update-ref',
|
|
'-m', f'west update: moving to {new_manifest_rev}',
|
|
QUAL_MANIFEST_REV, new_manifest_rev])
|
|
|
|
def _maybe_sha(rev):
|
|
# Return true if and only if the given revision might be a SHA.
|
|
|
|
try:
|
|
int(rev, 16)
|
|
except ValueError:
|
|
return False
|
|
|
|
return len(rev) <= 40
|
|
|
|
def _rev_type(project, rev=None):
|
|
# Returns a "refined" revision type of rev (default:
|
|
# project.revision) as one of the following strings: 'tag', 'tree',
|
|
# 'blob', 'commit', 'branch', 'other'.
|
|
#
|
|
# The approach combines git cat-file -t and git rev-parse because,
|
|
# while cat-file can for sure tell us a blob, tree, or tag, it
|
|
# doesn't have a way to disambiguate between branch names and
|
|
# other types of commit-ishes, like SHAs, things like "HEAD" or
|
|
# "HEAD~2", etc.
|
|
#
|
|
# We need this extra layer of refinement to be able to avoid
|
|
# fetching SHAs that are already available locally.
|
|
#
|
|
# This doesn't belong in manifest.py because it contains "west
|
|
# update" specific logic.
|
|
if not rev:
|
|
rev = project.revision
|
|
cp = project.git(['cat-file', '-t', rev], check=False,
|
|
capture_stdout=True, capture_stderr=True)
|
|
stdout = cp.stdout.decode('utf-8').strip()
|
|
if cp.returncode:
|
|
return 'other'
|
|
elif stdout in ('blob', 'tree', 'tag'):
|
|
return stdout
|
|
elif stdout != 'commit': # just future-proofing
|
|
return 'other'
|
|
|
|
# to tell branches apart from commits, we need rev-parse.
|
|
cp = project.git(['rev-parse', '--verify', '--symbolic-full-name', rev],
|
|
check=False, capture_stdout=True, capture_stderr=True)
|
|
if cp.returncode:
|
|
# This can happen if the ref name is ambiguous, e.g.:
|
|
#
|
|
# $ git update-ref ambiguous-ref HEAD~2
|
|
# $ git checkout -B ambiguous-ref
|
|
#
|
|
# Which creates both .git/ambiguous-ref and
|
|
# .git/refs/heads/ambiguous-ref.
|
|
return 'other'
|
|
|
|
stdout = cp.stdout.decode('utf-8').strip()
|
|
if stdout.startswith('refs/heads'):
|
|
return 'branch'
|
|
elif not stdout:
|
|
return 'commit'
|
|
else:
|
|
return 'other'
|
|
|
|
def _head_ok(project):
|
|
# Returns True if the reference 'HEAD' exists and is not a tag or remote
|
|
# ref (e.g. refs/remotes/origin/HEAD).
|
|
# Some versions of git will report 1, when doing
|
|
# 'git show-ref --verify HEAD' even if HEAD is valid, see #119.
|
|
# 'git show-ref --head <reference>' will always return 0 if HEAD or
|
|
# <reference> is valid.
|
|
# We are only interested in HEAD, thus we must avoid <reference> being
|
|
# valid. '/' can never point to valid reference, thus 'show-ref --head /'
|
|
# will return:
|
|
# - 0 if HEAD is present
|
|
# - 1 otherwise
|
|
return project.git('show-ref --quiet --head /',
|
|
check=False).returncode == 0
|
|
|
|
def _post_checkout_help(project, branch, sha, is_ancestor):
|
|
# Print helpful information to the user about a project that
|
|
# might have just left a branch behind.
|
|
|
|
if branch == 'HEAD':
|
|
# If there was no branch checked out, there are no
|
|
# additional diagnostics that need emitting.
|
|
return
|
|
|
|
rel = relpath(project.abspath)
|
|
if is_ancestor:
|
|
# If the branch we just left behind is a descendant of
|
|
# the new HEAD (e.g. if this is a topic branch the
|
|
# user is working on and the remote hasn't changed),
|
|
# print a message that makes it easy to get back,
|
|
# no matter where in the workspace os.getcwd() is.
|
|
log.wrn(f'left behind {project.name} branch "{branch}"; '
|
|
f'to switch back to it (fast forward):\n'
|
|
f' git -C {rel} checkout {branch}')
|
|
log.dbg('(To do this automatically in the future,',
|
|
'use "west update --keep-descendants".)')
|
|
else:
|
|
# Tell the user how they could rebase by hand, and
|
|
# point them at west update --rebase.
|
|
log.wrn(f'left behind {project.name} branch "{branch}"; '
|
|
f'to rebase onto the new HEAD:\n'
|
|
f' git -C {rel} rebase {sha} {branch}')
|
|
log.dbg('(To do this automatically in the future,',
|
|
'use "west update --rebase".)')
|
|
|
|
def projects_unknown(manifest, projects):
|
|
# Retrieve the projects with get_projects(project,
|
|
# only_cloned=False). Return a pair: (projects, unknown)
|
|
# containing either a projects list and None or None and a list of
|
|
# unknown project IDs.
|
|
|
|
try:
|
|
return (manifest.get_projects(projects, only_cloned=False), None)
|
|
except ValueError as ve:
|
|
if len(ve.args) != 2:
|
|
raise # not directly raised by get_projects()
|
|
unknown = ve.args[0]
|
|
if not unknown:
|
|
raise # only_cloned is False, so this "can't happen"
|
|
return (None, unknown)
|
|
|
|
def die_unknown(unknown):
|
|
# Scream and die about unknown projects.
|
|
|
|
s = 's' if len(unknown) > 1 else ''
|
|
names = ' '.join(unknown)
|
|
log.die(f'unknown project name{s}/path{s}: {names}\n'
|
|
' Hint: use "west list" to list all projects.')
|
|
|
|
#
|
|
# Special files and directories in the west workspace.
|
|
#
|
|
# These are given variable names for clarity, but they can't be
|
|
# changed without propagating the changes into west itself.
|
|
#
|
|
|
|
# Top-level west directory, containing west itself and the manifest.
|
|
WEST_DIR = '.west'
|
|
|
|
# Default manifest repository URL.
|
|
MANIFEST_URL_DEFAULT = 'https://github.com/zephyrproject-rtos/zephyr'
|
|
|
|
#
|
|
# Other shared globals.
|
|
#
|
|
|
|
ACTIVE_PROJECTS_HELP = '''\
|
|
ACTIVE PROJECTS
|
|
---------------
|
|
|
|
Default output is limited to "active" projects as determined by the:
|
|
|
|
- "group-filter" manifest file section
|
|
- "manifest.group-filter" local configuration option in .west/config
|
|
|
|
To include inactive projects as well, use "--all" or give an explicit
|
|
list of projects (by name or path). See the west documentation for
|
|
more details on active projects.
|
|
'''
|
|
|
|
ACTIVE_CLONED_PROJECTS_HELP = f'''\
|
|
{ACTIVE_PROJECTS_HELP}
|
|
|
|
Regardless of the above, output is limited to cloned projects.
|
|
'''
|
|
|
|
#
|
|
# Helper class for creating format string keys that are expensive or
|
|
# undesirable to compute if not needed.
|
|
#
|
|
|
|
class DelayFormat:
|
|
'''Delays formatting an object.'''
|
|
|
|
def __init__(self, obj):
|
|
'''Delay formatting `obj` until a format operation using it.
|
|
|
|
:param obj: object to format
|
|
|
|
If callable(obj) returns True, then obj() will be used as the
|
|
string to be formatted. Otherwise, str(obj) is used.'''
|
|
self.obj = obj
|
|
self.as_str = None
|
|
|
|
def __format__(self, format_spec):
|
|
if self.as_str is None:
|
|
if callable(self.obj):
|
|
self.as_str = self.obj()
|
|
assert isinstance(self.as_str, str)
|
|
else:
|
|
self.as_str = str(self.obj)
|
|
return ('{:' + format_spec + '}').format(self.as_str)
|
|
|
|
#
|
|
# Logging helpers
|
|
#
|
|
|
|
|
|
class ProjectCommandLogFormatter(logging.Formatter):
|
|
|
|
def __init__(self):
|
|
super().__init__(fmt='%(name)s: %(message)s')
|
|
|
|
|
|
class ProjectCommandLogHandler(logging.Handler):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.setFormatter(ProjectCommandLogFormatter())
|
|
|
|
def emit(self, record):
|
|
fmt = self.format(record)
|
|
lvl = record.levelno
|
|
if lvl > logging.CRITICAL:
|
|
log.die(fmt)
|
|
elif lvl >= logging.ERROR:
|
|
log.err(fmt)
|
|
elif lvl >= logging.WARNING:
|
|
log.wrn(fmt)
|
|
elif lvl >= logging.INFO:
|
|
log.inf(fmt)
|
|
elif lvl >= logging.DEBUG:
|
|
log.dbg(fmt)
|
|
else:
|
|
log.dbg(fmt, level=log.VERBOSE_EXTREME)
|