2016-01-04 16:33:20 +01:00
#!/usr/bin/env python3
2010-11-11 01:03:39 +01:00
#
# build.py - part of the FDroid server tools
2014-05-02 00:04:51 +02:00
# Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
2014-01-28 14:07:19 +01:00
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
2010-11-11 01:03:39 +01:00
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
2014-07-09 11:08:11 +02:00
import glob
2010-11-11 01:03:39 +01:00
import subprocess
2018-08-29 15:43:16 +02:00
import posixpath
2010-11-11 01:03:39 +01:00
import re
2011-01-03 00:26:12 +01:00
import tarfile
2018-01-15 01:03:47 +01:00
import threading
2012-02-03 17:01:35 +01:00
import traceback
2013-05-15 19:18:24 +02:00
import time
2017-04-20 17:48:38 +02:00
import requests
2017-04-04 18:58:16 +02:00
import tempfile
2019-01-29 11:23:45 +01:00
import argparse
2014-01-27 16:45:30 +01:00
import logging
2017-09-13 17:33:57 +02:00
from gettext import ngettext
2023-02-16 13:11:26 +01:00
from pathlib import Path
2010-11-11 01:03:39 +01:00
2017-09-13 18:03:57 +02:00
from . import _
2016-01-04 17:37:35 +01:00
from . import common
from . import net
from . import metadata
from . import scanner
2017-03-26 01:51:28 +01:00
from . import vmtools
2018-09-17 22:44:53 +02:00
from . common import FDroidPopen
2017-05-22 21:33:52 +02:00
from . exception import FDroidException , BuildException , VCSException
2012-02-21 01:01:07 +01:00
2014-05-02 00:04:51 +02:00
try :
import paramiko
2014-05-02 04:46:51 +02:00
except ImportError :
pass
2014-05-01 00:23:57 +02:00
2022-03-29 15:28:31 +02:00
buildserverid = None
2023-02-02 14:58:02 +01:00
ssh_channel = None
2022-03-29 15:28:31 +02:00
2014-05-02 05:39:33 +02:00
2014-05-02 00:04:51 +02:00
# Note that 'force' here also implies test mode.
2017-03-21 23:51:15 +01:00
def build_server ( app , build , vcs , build_dir , output_dir , log_dir , force ) :
""" Do a build on the builder vm.
2021-05-17 09:56:51 +02:00
Parameters
- - - - - - - - - -
app
app metadata dict
build
vcs
version control system controller object
build_dir
local source - code checkout of app
output_dir
target folder for the build result
force
2017-03-21 23:51:15 +01:00
"""
2023-02-02 14:58:02 +01:00
global buildserverid , ssh_channel
2017-02-09 23:49:42 +01:00
2014-05-02 04:46:51 +02:00
try :
paramiko
2018-06-27 01:05:45 +02:00
except NameError as e :
raise BuildException ( " Paramiko is required to use the buildserver " ) from e
2014-05-02 00:04:51 +02:00
if options . verbose :
2015-11-21 02:24:28 +01:00
logging . getLogger ( " paramiko " ) . setLevel ( logging . INFO )
2014-05-02 00:04:51 +02:00
else :
logging . getLogger ( " paramiko " ) . setLevel ( logging . WARN )
2020-10-31 10:55:14 +01:00
sshinfo = vmtools . get_clean_builder ( ' builder ' )
2014-05-02 00:04:51 +02:00
2018-06-12 16:18:21 +02:00
output = None
2014-05-02 00:04:51 +02:00
try :
2017-02-09 23:49:42 +01:00
if not buildserverid :
2018-06-27 01:05:45 +02:00
try :
buildserverid = subprocess . check_output ( [ ' vagrant ' , ' ssh ' , ' -c ' ,
' cat /home/vagrant/buildserverid ' ] ,
cwd = ' builder ' ) . strip ( ) . decode ( )
logging . debug ( _ ( ' Fetched buildserverid from VM: {buildserverid} ' )
. format ( buildserverid = buildserverid ) )
except Exception as e :
if type ( buildserverid ) is not str or not re . match ( ' ^[0-9a-f] {40} $ ' , buildserverid ) :
logging . info ( subprocess . check_output ( [ ' vagrant ' , ' status ' ] , cwd = " builder " ) )
raise FDroidException ( " Could not obtain buildserverid from buldserver VM. "
" (stored inside the buildserver VM at ' /home/vagrant/buildserverid ' ) "
" Please reset your buildserver, the setup VM is broken. " ) from e
2012-09-06 19:36:34 +02:00
# Open SSH connection...
2014-01-27 16:45:30 +01:00
logging . info ( " Connecting to virtual machine... " )
2014-04-03 18:14:59 +02:00
sshs = paramiko . SSHClient ( )
2023-12-14 14:28:48 +01:00
sshs . set_missing_host_key_policy ( paramiko . AutoAddPolicy ( ) ) # nosec B507 only connects to local VM
2014-05-02 00:04:51 +02:00
sshs . connect ( sshinfo [ ' hostname ' ] , username = sshinfo [ ' user ' ] ,
port = sshinfo [ ' port ' ] , timeout = 300 ,
look_for_keys = False , key_filename = sshinfo [ ' idfile ' ] )
2012-09-06 19:36:34 +02:00
2018-08-29 15:43:16 +02:00
homedir = posixpath . join ( ' /home ' , sshinfo [ ' user ' ] )
2014-05-02 00:24:54 +02:00
2012-09-06 19:36:34 +02:00
# Get an SFTP connection...
ftp = sshs . open_sftp ( )
2016-07-01 14:00:38 +02:00
ftp . get_channel ( ) . settimeout ( 60 )
2012-09-06 19:36:34 +02:00
# Put all the necessary files in place...
2014-05-02 00:24:54 +02:00
ftp . chdir ( homedir )
2012-09-06 19:36:34 +02:00
# Helper to copy the contents of a directory to the server...
def send_dir ( path ) :
2023-02-16 14:28:14 +01:00
logging . debug ( " rsyncing %s to %s " % ( path , ftp . getcwd ( ) ) )
2017-12-06 09:47:08 +01:00
# TODO this should move to `vagrant rsync` from >= v1.5
2017-12-05 21:31:55 +01:00
try :
2018-05-29 12:31:56 +02:00
subprocess . check_output ( [ ' rsync ' , ' --recursive ' , ' --perms ' , ' --links ' , ' --quiet ' , ' --rsh= '
+ ' ssh -o StrictHostKeyChecking=no '
+ ' -o UserKnownHostsFile=/dev/null '
+ ' -o LogLevel=FATAL '
+ ' -o IdentitiesOnly=yes '
+ ' -o PasswordAuthentication=no '
+ ' -p ' + str ( sshinfo [ ' port ' ] )
+ ' -i ' + sshinfo [ ' idfile ' ] ,
2017-12-05 21:31:55 +01:00
path ,
2017-12-05 21:37:04 +01:00
sshinfo [ ' user ' ] + " @ " + sshinfo [ ' hostname ' ] + " : " + ftp . getcwd ( ) ] ,
2017-12-05 21:31:55 +01:00
stderr = subprocess . STDOUT )
except subprocess . CalledProcessError as e :
2022-06-01 22:52:22 +02:00
raise FDroidException ( str ( e ) , e . output . decode ( ) ) from e
2012-09-06 19:36:34 +02:00
2014-01-27 16:45:30 +01:00
logging . info ( " Preparing server for build... " )
2012-09-06 19:36:34 +02:00
serverpath = os . path . abspath ( os . path . dirname ( __file__ ) )
2015-09-24 11:19:17 +02:00
ftp . mkdir ( ' fdroidserver ' )
ftp . chdir ( ' fdroidserver ' )
ftp . put ( os . path . join ( serverpath , ' .. ' , ' fdroid ' ) , ' fdroid ' )
2018-07-08 17:34:17 +02:00
ftp . put ( os . path . join ( serverpath , ' .. ' , ' gradlew-fdroid ' ) , ' gradlew-fdroid ' )
2018-08-29 14:54:37 +02:00
ftp . chmod ( ' fdroid ' , 0o755 ) # nosec B103 permissions are appropriate
ftp . chmod ( ' gradlew-fdroid ' , 0o755 ) # nosec B103 permissions are appropriate
2015-09-24 11:19:17 +02:00
send_dir ( os . path . join ( serverpath ) )
ftp . chdir ( homedir )
2014-02-17 19:16:14 +01:00
ftp . put ( os . path . join ( serverpath , ' .. ' , ' buildserver ' ,
2020-10-22 23:34:47 +02:00
' config.buildserver.yml ' ) , ' config.yml ' )
ftp . chmod ( ' config.yml ' , 0o600 )
2012-09-06 19:36:34 +02:00
2014-04-10 14:58:42 +02:00
# Copy over the ID (head commit hash) of the fdroidserver in use...
2018-01-23 23:56:15 +01:00
with open ( os . path . join ( os . getcwd ( ) , ' tmp ' , ' fdroidserverid ' ) , ' wb ' ) as fp :
fp . write ( subprocess . check_output ( [ ' git ' , ' rev-parse ' , ' HEAD ' ] ,
cwd = serverpath ) )
2014-04-10 14:58:42 +02:00
ftp . put ( ' tmp/fdroidserverid ' , ' fdroidserverid ' )
2012-09-06 19:36:34 +02:00
# Copy the metadata - just the file for this app...
ftp . mkdir ( ' metadata ' )
2013-05-22 11:17:02 +02:00
ftp . mkdir ( ' srclibs ' )
2012-09-06 19:36:34 +02:00
ftp . chdir ( ' metadata ' )
2017-05-16 16:28:24 +02:00
ftp . put ( app . metadatapath , os . path . basename ( app . metadatapath ) )
2012-09-08 10:56:20 +02:00
# And patches if there are any...
2015-11-28 13:09:47 +01:00
if os . path . exists ( os . path . join ( ' metadata ' , app . id ) ) :
send_dir ( os . path . join ( ' metadata ' , app . id ) )
2013-05-22 11:17:02 +02:00
2014-05-02 00:24:54 +02:00
ftp . chdir ( homedir )
2012-09-06 19:36:34 +02:00
# Create the build directory...
ftp . mkdir ( ' build ' )
2012-08-31 15:48:50 +02:00
ftp . chdir ( ' build ' )
2012-09-06 19:36:34 +02:00
ftp . mkdir ( ' extlib ' )
2013-05-20 22:00:31 +02:00
ftp . mkdir ( ' srclib ' )
2012-09-06 19:36:34 +02:00
# Copy any extlibs that are required...
2015-11-28 17:55:27 +01:00
if build . extlibs :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' build ' , ' extlib ' ) )
2015-11-28 17:55:27 +01:00
for lib in build . extlibs :
2013-09-11 13:45:02 +02:00
lib = lib . strip ( )
2013-11-12 21:14:16 +01:00
libsrc = os . path . join ( ' build/extlib ' , lib )
if not os . path . exists ( libsrc ) :
raise BuildException ( " Missing extlib {0} " . format ( libsrc ) )
2012-09-06 19:36:34 +02:00
lp = lib . split ( ' / ' )
for d in lp [ : - 1 ] :
2013-03-05 20:47:52 +01:00
if d not in ftp . listdir ( ) :
ftp . mkdir ( d )
2012-09-06 19:36:34 +02:00
ftp . chdir ( d )
2013-11-12 21:14:16 +01:00
ftp . put ( libsrc , lp [ - 1 ] )
2017-09-13 18:03:57 +02:00
for _ignored in lp [ : - 1 ] :
2012-09-06 19:36:34 +02:00
ftp . chdir ( ' .. ' )
# Copy any srclibs that are required...
2012-09-08 10:56:20 +02:00
srclibpaths = [ ]
2015-11-28 17:55:27 +01:00
if build . srclibs :
for lib in build . srclibs :
2014-05-01 00:23:57 +02:00
srclibpaths . append (
2015-06-03 13:51:41 +02:00
common . getsrclib ( lib , ' build/srclib ' , basepath = True , prepare = False ) )
2013-11-18 10:24:02 +01:00
2012-09-08 10:56:20 +02:00
# If one was used for the main source, add that too.
basesrclib = vcs . getsrclib ( )
if basesrclib :
srclibpaths . append ( basesrclib )
2013-11-16 22:49:37 +01:00
for name , number , lib in srclibpaths :
2014-01-27 16:45:30 +01:00
logging . info ( " Sending srclib ' %s ' " % lib )
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' build ' , ' srclib ' ) )
2012-09-20 15:16:08 +02:00
if not os . path . exists ( lib ) :
raise BuildException ( " Missing srclib directory ' " + lib + " ' " )
2013-07-18 22:56:37 +02:00
fv = ' .fdroidvcs- ' + name
ftp . put ( os . path . join ( ' build/srclib ' , fv ) , fv )
2012-09-08 10:56:20 +02:00
send_dir ( lib )
2013-05-22 11:17:02 +02:00
# Copy the metadata file too...
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' srclibs ' ) )
2020-08-24 16:23:13 +02:00
srclibsfile = os . path . join ( ' srclibs ' , name + ' .yml ' )
if os . path . isfile ( srclibsfile ) :
ftp . put ( srclibsfile , os . path . basename ( srclibsfile ) )
2020-04-16 11:32:37 +02:00
else :
2020-08-24 16:23:13 +02:00
raise BuildException ( _ ( ' cannot find required srclibs: " {path} " ' )
. format ( path = srclibsfile ) )
2013-07-18 22:56:37 +02:00
# Copy the main app source code
# (no need if it's a srclib)
if ( not basesrclib ) and os . path . exists ( build_dir ) :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' build ' ) )
2015-11-28 13:09:47 +01:00
fv = ' .fdroidvcs- ' + app . id
2013-07-18 22:56:37 +02:00
ftp . put ( os . path . join ( ' build ' , fv ) , fv )
send_dir ( build_dir )
2012-09-06 19:36:34 +02:00
# Execute the build script...
2014-01-27 16:45:30 +01:00
logging . info ( " Starting build... " )
2023-02-02 14:57:47 +01:00
ssh_channel = sshs . get_transport ( ) . open_session ( )
ssh_channel . get_pty ( )
2018-08-29 15:43:16 +02:00
cmdline = posixpath . join ( homedir , ' fdroidserver ' , ' fdroid ' )
2015-09-24 11:19:17 +02:00
cmdline + = ' build --on-server '
2012-09-27 20:58:03 +02:00
if force :
cmdline + = ' --force --test '
2013-11-24 11:29:09 +01:00
if options . verbose :
cmdline + = ' --verbose '
2024-02-16 11:43:13 +01:00
if options . refresh_scanner or config . get ( ' refresh_scanner ' ) :
cmdline + = ' --refresh-scanner '
2017-03-01 17:27:17 +01:00
if options . skipscan :
cmdline + = ' --skip-scan '
2018-07-09 14:48:59 +02:00
if options . notarball :
cmdline + = ' --no-tarball '
2022-06-03 12:41:47 +02:00
if ( options . scan_binary or config . get ( ' scan_binary ' ) ) and not options . skipscan :
cmdline + = ' --scan-binary '
2016-11-23 17:52:04 +01:00
cmdline + = " %s : %s " % ( app . id , build . versionCode )
2023-02-02 14:57:47 +01:00
ssh_channel . exec_command ( ' bash --login -c " ' + cmdline + ' " ' ) # nosec B601 inputs are sanitized
2017-02-09 23:48:40 +01:00
2017-09-14 01:38:34 +02:00
# Fetch build process output ...
try :
2023-02-02 14:57:47 +01:00
cmd_stdout = ssh_channel . makefile ( ' rb ' , 1024 )
2017-09-14 01:38:34 +02:00
output = bytes ( )
2021-05-26 17:35:39 +02:00
output + = common . get_android_tools_version_log ( ) . encode ( )
2023-02-02 14:57:47 +01:00
while not ssh_channel . exit_status_ready ( ) :
2017-09-14 01:38:34 +02:00
line = cmd_stdout . readline ( )
if line :
if options . verbose :
Ignore encoding errors in logs
Found while compiling io.privatestorage.privatestoragemobile_9:
Traceback (most recent call last):
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 1053, in main
if trybuild(app, build, build_dir, output_dir, log_dir,
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 813, in trybuild
build_server(app, build, vcs, build_dir, output_dir, log_dir, force)
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 253, in build_server
logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xfa in position 8: invalid start byte
2023-09-22 12:50:05 +02:00
logging . debug ( " buildserver > " + str ( line , ' utf-8 ' , ' replace ' ) . rstrip ( ) )
2017-09-14 01:38:34 +02:00
output + = line
else :
time . sleep ( 0.05 )
for line in cmd_stdout . readlines ( ) :
if options . verbose :
Ignore encoding errors in logs
Found while compiling io.privatestorage.privatestoragemobile_9:
Traceback (most recent call last):
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 1053, in main
if trybuild(app, build, build_dir, output_dir, log_dir,
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 813, in trybuild
build_server(app, build, vcs, build_dir, output_dir, log_dir, force)
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 253, in build_server
logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xfa in position 8: invalid start byte
2023-09-22 12:50:05 +02:00
logging . debug ( " buildserver > " + str ( line , ' utf-8 ' , ' replace ' ) . rstrip ( ) )
2017-09-14 01:38:34 +02:00
output + = line
finally :
cmd_stdout . close ( )
# Check build process exit status ...
2014-01-27 16:45:30 +01:00
logging . info ( " ...getting exit status " )
2023-02-02 14:57:47 +01:00
returncode = ssh_channel . recv_exit_status ( )
2012-09-06 19:36:34 +02:00
if returncode != 0 :
2018-01-20 22:16:39 +01:00
if timeout_event . is_set ( ) :
message = " Timeout exceeded! Build VM force-stopped for {0} : {1} "
else :
message = " Build.py failed on server for {0} : {1} "
raise BuildException ( message . format ( app . id , build . versionName ) ,
Ignore encoding errors in logs
Found while compiling io.privatestorage.privatestoragemobile_9:
Traceback (most recent call last):
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 1053, in main
if trybuild(app, build, build_dir, output_dir, log_dir,
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 813, in trybuild
build_server(app, build, vcs, build_dir, output_dir, log_dir, force)
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 253, in build_server
logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xfa in position 8: invalid start byte
2023-09-22 12:50:05 +02:00
str ( output , ' utf-8 ' , ' replace ' ) )
2012-09-06 19:36:34 +02:00
2017-03-21 23:51:15 +01:00
# Retreive logs...
toolsversion_log = common . get_toolsversion_logname ( app , build )
try :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , log_dir ) )
2017-03-21 23:51:15 +01:00
ftp . get ( toolsversion_log , os . path . join ( log_dir , toolsversion_log ) )
logging . debug ( ' retrieved %s ' , toolsversion_log )
2017-03-22 14:01:32 +01:00
except Exception as e :
2020-05-25 17:36:58 +02:00
logging . warning ( ' could not get %s from builder vm: %s ' % ( toolsversion_log , e ) )
2017-03-21 23:51:15 +01:00
2012-09-06 19:36:34 +02:00
# Retrieve the built files...
2014-01-27 16:45:30 +01:00
logging . info ( " Retrieving build output... " )
2013-01-22 10:49:33 +01:00
if force :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' tmp ' ) )
2012-09-27 20:58:03 +02:00
else :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' unsigned ' ) )
2016-10-31 16:51:34 +01:00
apkfile = common . get_release_filename ( app , build )
2015-11-28 17:55:27 +01:00
tarball = common . getsrcname ( app , build )
2012-09-06 19:36:34 +02:00
try :
ftp . get ( apkfile , os . path . join ( output_dir , apkfile ) )
2014-01-28 22:13:18 +01:00
if not options . notarball :
ftp . get ( tarball , os . path . join ( output_dir , tarball ) )
2022-06-01 22:52:22 +02:00
except Exception as exc :
2014-05-02 00:04:51 +02:00
raise BuildException (
2017-09-14 01:38:34 +02:00
" Build failed for {0} : {1} - missing output files " . format (
Ignore encoding errors in logs
Found while compiling io.privatestorage.privatestoragemobile_9:
Traceback (most recent call last):
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 1053, in main
if trybuild(app, build, build_dir, output_dir, log_dir,
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 813, in trybuild
build_server(app, build, vcs, build_dir, output_dir, log_dir, force)
File "/home/fdroid/fdroidserver/fdroidserver/build.py", line 253, in build_server
logging.debug("buildserver > " + str(line, 'utf-8').rstrip())
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xfa in position 8: invalid start byte
2023-09-22 12:50:05 +02:00
app . id , build . versionName ) , str ( output , ' utf-8 ' , ' replace ' ) ) from exc
2012-09-06 19:36:34 +02:00
ftp . close ( )
finally :
# Suspend the build server.
2017-04-14 15:58:16 +02:00
vm = vmtools . get_build_vm ( ' builder ' )
2020-10-31 10:55:14 +01:00
logging . info ( ' destroying buildserver after build ' )
vm . destroy ( )
2012-02-21 01:01:07 +01:00
2018-06-12 16:18:21 +02:00
# deploy logfile to repository web server
if output :
2018-06-23 23:26:42 +02:00
common . deploy_build_log_with_rsync ( app . id , build . versionCode , output )
2018-06-12 16:18:21 +02:00
else :
logging . debug ( ' skip publishing full build logs: '
' no output present ' )
2014-05-01 00:23:57 +02:00
2016-06-20 20:00:59 +02:00
def force_gradle_build_tools ( build_dir , build_tools ) :
2014-01-11 13:05:02 +01:00
for root , dirs , files in os . walk ( build_dir ) :
2015-04-06 17:42:26 +02:00
for filename in files :
if not filename . endswith ( ' .gradle ' ) :
continue
2015-03-31 23:08:32 +02:00
path = os . path . join ( root , filename )
2015-04-17 13:02:47 +02:00
if not os . path . isfile ( path ) :
continue
2016-06-20 20:00:59 +02:00
logging . debug ( " Forcing build-tools %s in %s " % ( build_tools , path ) )
2015-10-14 19:34:30 +02:00
common . regsub_file ( r """ ( \ s*)buildToolsVersion([ \ s=]+).* """ ,
2016-06-20 20:00:59 +02:00
r """ \ 1buildToolsVersion \ 2 ' %s ' """ % build_tools ,
2015-07-30 22:13:12 +02:00
path )
2013-08-22 18:28:30 +02:00
2012-02-21 01:01:07 +01:00
2017-08-26 12:55:34 +02:00
def transform_first_char ( string , method ) :
2021-06-07 12:26:57 +02:00
""" Use method() on the first character of string. """
2015-01-02 22:26:22 +01:00
if len ( string ) == 0 :
return string
if len ( string ) == 1 :
2017-08-26 12:55:34 +02:00
return method ( string )
return method ( string [ 0 ] ) + string [ 1 : ]
2015-01-02 22:26:22 +01:00
2017-04-13 14:18:48 +02:00
def get_metadata_from_apk ( app , build , apkfile ) :
2021-06-07 12:26:57 +02:00
""" Get the required metadata from the built APK.
2017-04-13 14:18:48 +02:00
2021-06-07 12:26:57 +02:00
VersionName is allowed to be a blank string , i . e . ' '
2018-09-17 22:44:53 +02:00
"""
appid , versionCode , versionName = common . get_apk_id ( apkfile )
native_code = common . get_native_code ( apkfile )
2016-10-31 19:53:55 +01:00
2018-09-17 22:44:53 +02:00
if build . buildjni and build . buildjni != [ ' no ' ] and not native_code :
raise BuildException ( " Native code should have been built but none was packaged " )
2016-10-31 19:53:55 +01:00
if build . novcheck :
2018-09-17 22:44:53 +02:00
versionCode = build . versionCode
versionName = build . versionName
if not versionCode or versionName is None :
2016-10-31 19:53:55 +01:00
raise BuildException ( " Could not find version information in build in output " )
2018-09-17 22:44:53 +02:00
if not appid :
2016-10-31 19:53:55 +01:00
raise BuildException ( " Could not find package ID in output " )
2018-09-17 22:44:53 +02:00
if appid != app . id :
raise BuildException ( " Wrong package ID - build " + appid + " but expected " + app . id )
2016-10-31 19:53:55 +01:00
2018-09-17 22:44:53 +02:00
return versionCode , versionName
2016-10-31 19:53:55 +01:00
2017-03-21 23:51:15 +01:00
def build_local ( app , build , vcs , build_dir , output_dir , log_dir , srclib_dir , extlib_dir , tmp_dir , force , onserver , refresh ) :
2012-02-26 15:09:25 +01:00
""" Do a build locally. """
2015-11-28 17:55:27 +01:00
ndk_path = build . ndk_path ( )
2016-07-13 17:32:52 +02:00
if build . ndk or ( build . buildjni and build . buildjni != [ ' no ' ] ) :
2015-11-28 17:55:27 +01:00
if not ndk_path :
2021-05-28 09:23:16 +02:00
logging . warning ( " Android NDK version ' %s ' could not be found! " % build . ndk )
logging . warning ( " Configured versions: " )
2016-01-04 17:02:28 +01:00
for k , v in config [ ' ndk_paths ' ] . items ( ) :
2015-01-03 00:02:54 +01:00
if k . endswith ( " _orig " ) :
continue
2021-05-28 09:23:16 +02:00
logging . warning ( " %s : %s " % ( k , v ) )
2021-05-12 11:28:25 +02:00
if onserver :
common . auto_install_ndk ( build )
else :
raise FDroidException ( )
2015-11-28 17:55:27 +01:00
elif not os . path . isdir ( ndk_path ) :
logging . critical ( " Android NDK ' %s ' is not a directory! " % ndk_path )
2017-05-22 21:33:52 +02:00
raise FDroidException ( )
2014-02-17 20:03:55 +01:00
2015-08-05 14:39:58 +02:00
common . set_FDroidPopen_env ( build )
2015-01-06 19:41:55 +01:00
2017-03-21 23:51:15 +01:00
# create ..._toolsversion.log when running in builder vm
if onserver :
2017-06-28 23:01:45 +02:00
# before doing anything, run the sudo commands to setup the VM
if build . sudo :
logging . info ( " Running ' sudo ' commands in %s " % os . getcwd ( ) )
2020-02-11 19:44:06 +01:00
p = FDroidPopen ( [ ' sudo ' , ' DEBIAN_FRONTEND=noninteractive ' ,
2022-09-09 12:36:54 +02:00
' bash ' , ' -e ' , ' -u ' , ' -o ' , ' pipefail ' , ' -x ' , ' -c ' , ' ; ' . join ( build . sudo ) ] )
2017-06-28 23:01:45 +02:00
if p . returncode != 0 :
raise BuildException ( " Error running sudo command for %s : %s " %
( app . id , build . versionName ) , p . output )
build: force purging of sudo, ignore error message
Fixes bb758d3f, spotted by @bubu:
DEBUG: buildserver > DEBUG: > sudo apt-get -y purge sudo
DEBUG: buildserver > Reading package lists...
DEBUG: buildserver > Building dependency tree...
DEBUG: buildserver > Reading state information...
DEBUG: buildserver > The following packages will be REMOVED:
DEBUG: buildserver > sudo*
DEBUG: buildserver > 0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded.
DEBUG: buildserver > After this operation, 2,391 kB disk space will be freed.
(Reading database ... 68491 files and directories currently installed.)
DEBUG: buildserver > Removing sudo (1.8.10p3-1+deb8u4) ...
DEBUG: buildserver > You have asked that the sudo package be removed,
DEBUG: buildserver > but no root password has been set.
DEBUG: buildserver > Without sudo, you may not be able to gain administrative privileges.
DEBUG: buildserver >
DEBUG: buildserver > If you would prefer to access the root account with su(1)
DEBUG: buildserver > or by logging in directly,
DEBUG: buildserver > you must set a root password with "sudo passwd".
DEBUG: buildserver >
DEBUG: buildserver > If you have arranged other means to access the root account,
DEBUG: buildserver > and you are sure this is what you want,
DEBUG: buildserver > you may bypass this check by setting an environment variable
DEBUG: buildserver > (export SUDO_FORCE_REMOVE=yes).
DEBUG: buildserver >
DEBUG: buildserver > Refusing to remove sudo.
DEBUG: buildserver > dpkg: error processing package sudo (--purge):
DEBUG: buildserver > subprocess installed pre-removal script returned error exit status 1
DEBUG: buildserver > Errors were encountered while processing:
DEBUG: buildserver > sudo
DEBUG: buildserver > E: Sub-process /usr/bin/dpkg returned an error code (1)
2017-12-14 14:42:09 +01:00
p = FDroidPopen ( [ ' sudo ' , ' passwd ' , ' --lock ' , ' root ' ] )
if p . returncode != 0 :
raise BuildException ( " Error locking root account for %s : %s " %
( app . id , build . versionName ) , p . output )
2017-12-18 10:02:19 +01:00
p = FDroidPopen ( [ ' sudo ' , ' SUDO_FORCE_REMOVE=yes ' , ' dpkg ' , ' --purge ' , ' sudo ' ] )
2017-12-07 22:26:13 +01:00
if p . returncode != 0 :
raise BuildException ( " Error removing sudo for %s : %s " %
( app . id , build . versionName ) , p . output )
2017-03-21 23:51:15 +01:00
log_path = os . path . join ( log_dir ,
common . get_toolsversion_logname ( app , build ) )
with open ( log_path , ' w ' ) as f :
2021-05-26 17:35:39 +02:00
f . write ( common . get_android_tools_version_log ( ) )
2017-06-28 23:01:45 +02:00
else :
if build . sudo :
2019-01-15 15:14:09 +01:00
logging . warning ( ' %s : %s runs this on the buildserver with sudo: \n \t %s \n These commands were skipped because fdroid build is not running on a dedicated build server. '
2017-06-28 23:01:45 +02:00
% ( app . id , build . versionName , build . sudo ) )
2017-03-21 23:51:15 +01:00
2012-02-21 01:01:07 +01:00
# Prepare the source code...
2015-11-28 17:55:27 +01:00
root_dir , srclibpaths = common . prepare_source ( vcs , app , build ,
2014-05-06 19:50:52 +02:00
build_dir , srclib_dir ,
2015-07-14 12:32:39 +02:00
extlib_dir , onserver , refresh )
2012-02-21 01:01:07 +01:00
2013-10-10 15:48:39 +02:00
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
2015-01-02 22:26:22 +01:00
gradletasks = [ ]
2016-02-15 13:01:38 +01:00
bmethod = build . build_method ( )
if bmethod == ' maven ' :
2014-01-27 16:45:30 +01:00
logging . info ( " Cleaning Maven project... " )
2013-10-31 16:37:39 +01:00
cmd = [ config [ ' mvn3 ' ] , ' clean ' , ' -Dandroid.sdk.path= ' + config [ ' sdk_path ' ] ]
2013-10-10 15:48:39 +02:00
2015-11-28 17:55:27 +01:00
if ' @ ' in build . maven :
maven_dir = os . path . join ( root_dir , build . maven . split ( ' @ ' , 1 ) [ 1 ] )
2013-12-07 13:15:51 +01:00
maven_dir = os . path . normpath ( maven_dir )
2013-10-23 19:52:17 +02:00
else :
maven_dir = root_dir
2013-11-01 12:10:57 +01:00
p = FDroidPopen ( cmd , cwd = maven_dir )
2014-01-10 20:39:39 +01:00
2016-02-15 13:01:38 +01:00
elif bmethod == ' gradle ' :
2014-01-16 00:20:26 +01:00
2014-01-27 16:45:30 +01:00
logging . info ( " Cleaning Gradle project... " )
2015-01-02 22:26:22 +01:00
2015-11-28 17:55:27 +01:00
if build . preassemble :
gradletasks + = build . preassemble
2015-01-02 22:26:22 +01:00
2015-11-28 17:55:27 +01:00
flavours = build . gradle
2015-01-02 22:26:22 +01:00
if flavours == [ ' yes ' ] :
flavours = [ ]
2017-08-26 12:55:34 +02:00
flavours_cmd = ' ' . join ( [ transform_first_char ( flav , str . upper ) for flav in flavours ] )
2015-01-02 22:26:22 +01:00
gradletasks + = [ ' assemble ' + flavours_cmd + ' Release ' ]
2013-10-10 15:48:39 +02:00
2015-01-02 22:26:22 +01:00
cmd = [ config [ ' gradle ' ] ]
2015-11-28 17:55:27 +01:00
if build . gradleprops :
2016-02-15 20:22:17 +01:00
cmd + = [ ' -P ' + kv for kv in build . gradleprops ]
2015-08-25 00:54:05 +02:00
2015-04-10 00:14:52 +02:00
cmd + = [ ' clean ' ]
2018-07-08 17:34:17 +02:00
p = FDroidPopen ( cmd , cwd = root_dir , envs = { " GRADLE_VERSION_DIR " : config [ ' gradle_version_dir ' ] , " CACHEDIR " : config [ ' cachedir ' ] } )
2014-01-10 20:39:39 +01:00
2016-02-15 13:01:38 +01:00
elif bmethod == ' ant ' :
2014-01-27 16:45:30 +01:00
logging . info ( " Cleaning Ant project... " )
2014-01-10 20:39:39 +01:00
p = FDroidPopen ( [ ' ant ' , ' clean ' ] , cwd = root_dir )
2013-10-16 23:17:51 +02:00
if p is not None and p . returncode != 0 :
raise BuildException ( " Error cleaning %s : %s " %
2016-11-23 17:52:04 +01:00
( app . id , build . versionName ) , p . output )
2013-10-10 17:52:17 +02:00
2014-03-05 12:32:36 +01:00
for root , dirs , files in os . walk ( build_dir ) :
2015-10-08 12:08:21 +02:00
def del_dirs ( dl ) :
for d in dl :
2020-05-21 07:50:08 +02:00
shutil . rmtree ( os . path . join ( root , d ) , ignore_errors = True )
2015-10-08 12:08:21 +02:00
def del_files ( fl ) :
for f in fl :
if f in files :
os . remove ( os . path . join ( root , f ) )
2019-12-23 02:02:28 +01:00
if any ( f in files for f in [ ' build.gradle ' , ' build.gradle.kts ' , ' settings.gradle ' , ' settings.gradle.kts ' ] ) :
2015-10-08 12:08:21 +02:00
# Even when running clean, gradle stores task/artifact caches in
# .gradle/ as binary files. To avoid overcomplicating the scanner,
# manually delete them, just like `gradle clean` should have removed
2018-01-10 19:06:32 +01:00
# the build/* dirs.
del_dirs ( [ os . path . join ( ' build ' , ' android-profile ' ) ,
os . path . join ( ' build ' , ' generated ' ) ,
os . path . join ( ' build ' , ' intermediates ' ) ,
os . path . join ( ' build ' , ' outputs ' ) ,
os . path . join ( ' build ' , ' reports ' ) ,
os . path . join ( ' build ' , ' tmp ' ) ,
2020-05-21 07:50:35 +02:00
os . path . join ( ' buildSrc ' , ' build ' ) ,
2018-01-10 19:06:32 +01:00
' .gradle ' ] )
2015-10-08 12:08:21 +02:00
del_files ( [ ' gradlew ' , ' gradlew.bat ' ] )
if ' pom.xml ' in files :
del_dirs ( [ ' target ' ] )
if any ( f in files for f in [ ' ant.properties ' , ' project.properties ' , ' build.xml ' ] ) :
del_dirs ( [ ' bin ' , ' gen ' ] )
if ' jni ' in dirs :
del_dirs ( [ ' obj ' ] )
2014-03-05 12:32:36 +01:00
2015-01-06 14:40:31 +01:00
if options . skipscan :
2015-11-28 17:55:27 +01:00
if build . scandelete :
2015-01-06 14:40:31 +01:00
raise BuildException ( " Refusing to skip source scan since scandelete is present " )
else :
2014-02-16 17:40:54 +01:00
# Scan before building...
logging . info ( " Scanning source for common problems... " )
2020-06-03 18:34:47 +02:00
scanner . options = options # pass verbose through
2017-05-25 20:11:14 +02:00
count = scanner . scan_source ( build_dir , build )
2014-04-03 16:04:06 +02:00
if count > 0 :
if force :
2017-09-13 17:33:57 +02:00
logging . warning ( ngettext ( ' Scanner found {} problem ' ,
' Scanner found {} problems ' , count ) . format ( count ) )
2014-04-03 16:04:06 +02:00
else :
2017-09-13 17:33:57 +02:00
raise BuildException ( ngettext (
" Can ' t build due to {} error while scanning " ,
" Can ' t build due to {} errors while scanning " , count ) . format ( count ) )
2012-02-21 01:01:07 +01:00
2014-01-28 22:13:18 +01:00
if not options . notarball :
# Build the source tarball right before we build the release...
2014-01-29 13:57:57 +01:00
logging . info ( " Creating source tarball... " )
2015-11-28 17:55:27 +01:00
tarname = common . getsrcname ( app , build )
2014-01-28 22:13:18 +01:00
tarball = tarfile . open ( os . path . join ( tmp_dir , tarname ) , " w:gz " )
2014-05-01 00:23:57 +02:00
2018-07-25 15:31:28 +02:00
def tarexc ( t ) :
return None if any ( t . name . endswith ( s ) for s in [ ' .svn ' , ' .git ' , ' .hg ' , ' .bzr ' ] ) else t
tarball . add ( build_dir , tarname , filter = tarexc )
2014-01-28 22:13:18 +01:00
tarball . close ( )
2012-02-21 01:01:07 +01:00
2013-03-20 10:30:56 +01:00
# Run a build command if one is required...
2015-11-28 17:55:27 +01:00
if build . build :
2014-05-31 23:10:16 +02:00
logging . info ( " Running ' build ' commands in %s " % root_dir )
2022-09-09 12:36:54 +02:00
cmd = common . replace_config_vars ( " ; " . join ( build . build ) , build )
2014-05-31 23:10:16 +02:00
2013-06-15 19:21:28 +02:00
# Substitute source library paths into commands...
2013-11-16 22:49:37 +01:00
for name , number , libpath in srclibpaths :
2020-06-24 20:50:21 +02:00
cmd = cmd . replace ( ' $$ ' + name + ' $$ ' , os . path . join ( os . getcwd ( ) , libpath ) )
2013-10-09 23:36:24 +02:00
2022-09-09 12:36:54 +02:00
p = FDroidPopen ( [ ' bash ' , ' -e ' , ' -u ' , ' -o ' , ' pipefail ' , ' -x ' , ' -c ' , cmd ] , cwd = root_dir )
2013-12-30 17:04:16 +01:00
2013-03-20 10:30:56 +01:00
if p . returncode != 0 :
2013-10-09 23:36:24 +02:00
raise BuildException ( " Error running build command for %s : %s " %
2016-11-23 17:52:04 +01:00
( app . id , build . versionName ) , p . output )
2013-03-20 10:30:56 +01:00
2012-02-21 01:01:07 +01:00
# Build native stuff if required...
2015-11-28 17:55:27 +01:00
if build . buildjni and build . buildjni != [ ' no ' ] :
2014-05-31 23:10:16 +02:00
logging . info ( " Building the native code " )
2015-11-28 17:55:27 +01:00
jni_components = build . buildjni
2014-05-31 23:10:16 +02:00
2014-02-15 10:56:35 +01:00
if jni_components == [ ' yes ' ] :
2012-03-07 07:46:56 +01:00
jni_components = [ ' ' ]
2015-11-28 17:55:27 +01:00
cmd = [ os . path . join ( ndk_path , " ndk-build " ) , " -j1 " ]
2012-03-07 07:46:56 +01:00
for d in jni_components :
2014-04-03 16:10:54 +02:00
if d :
logging . info ( " Building native code in ' %s ' " % d )
else :
logging . info ( " Building native code in the main project " )
2015-09-15 03:12:15 +02:00
manifest = os . path . join ( root_dir , d , ' AndroidManifest.xml ' )
2013-03-31 19:33:18 +02:00
if os . path . exists ( manifest ) :
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open ( manifest , ' U ' ) . read ( )
open ( manifest , ' w ' ) . write ( manifest_text )
# In case the AM.xml read was big, free the memory
del manifest_text
2014-05-01 00:23:57 +02:00
p = FDroidPopen ( cmd , cwd = os . path . join ( root_dir , d ) )
2013-10-16 23:17:51 +02:00
if p . returncode != 0 :
2016-11-23 17:52:04 +01:00
raise BuildException ( " NDK build failed for %s : %s " % ( app . id , build . versionName ) , p . output )
2012-02-21 01:01:07 +01:00
2013-09-04 16:45:19 +02:00
p = None
2012-02-21 01:01:07 +01:00
# Build the release...
2016-02-15 13:01:38 +01:00
if bmethod == ' maven ' :
2014-01-27 17:04:22 +01:00
logging . info ( " Building Maven project... " )
2013-10-23 19:52:17 +02:00
2015-11-28 17:55:27 +01:00
if ' @ ' in build . maven :
maven_dir = os . path . join ( root_dir , build . maven . split ( ' @ ' , 1 ) [ 1 ] )
2013-10-23 19:52:17 +02:00
else :
maven_dir = root_dir
2013-12-11 17:29:38 +01:00
mvncmd = [ config [ ' mvn3 ' ] , ' -Dandroid.sdk.path= ' + config [ ' sdk_path ' ] ,
2014-05-06 19:50:52 +02:00
' -Dmaven.jar.sign.skip=true ' , ' -Dmaven.test.skip=true ' ,
' -Dandroid.sign.debug=false ' , ' -Dandroid.release=true ' ,
' package ' ]
2015-11-28 17:55:27 +01:00
if build . target :
target = build . target . split ( ' - ' ) [ 1 ]
2015-07-30 22:13:12 +02:00
common . regsub_file ( r ' <platform>[0-9]*</platform> ' ,
r ' <platform> %s </platform> ' % target ,
os . path . join ( root_dir , ' pom.xml ' ) )
2015-11-28 17:55:27 +01:00
if ' @ ' in build . maven :
2015-07-30 22:13:12 +02:00
common . regsub_file ( r ' <platform>[0-9]*</platform> ' ,
r ' <platform> %s </platform> ' % target ,
os . path . join ( maven_dir , ' pom.xml ' ) )
2013-06-10 22:50:15 +02:00
2013-11-04 20:19:31 +01:00
p = FDroidPopen ( mvncmd , cwd = maven_dir )
2013-09-02 19:05:13 +02:00
2013-11-09 12:21:43 +01:00
bindir = os . path . join ( root_dir , ' target ' )
2016-02-15 13:01:38 +01:00
elif bmethod == ' gradle ' :
2014-01-27 17:04:22 +01:00
logging . info ( " Building Gradle project... " )
2013-08-22 18:28:30 +02:00
2015-08-25 00:54:05 +02:00
cmd = [ config [ ' gradle ' ] ]
2015-11-28 17:55:27 +01:00
if build . gradleprops :
2016-02-15 20:22:17 +01:00
cmd + = [ ' -P ' + kv for kv in build . gradleprops ]
2015-08-25 00:54:05 +02:00
cmd + = gradletasks
2015-01-02 22:26:22 +01:00
2018-07-08 17:34:17 +02:00
p = FDroidPopen ( cmd , cwd = root_dir , envs = { " GRADLE_VERSION_DIR " : config [ ' gradle_version_dir ' ] , " CACHEDIR " : config [ ' cachedir ' ] } )
2013-09-02 19:05:13 +02:00
2016-02-15 13:01:38 +01:00
elif bmethod == ' ant ' :
2014-01-27 17:04:22 +01:00
logging . info ( " Building Ant project... " )
2013-10-16 23:17:51 +02:00
cmd = [ ' ant ' ]
2015-11-28 17:55:27 +01:00
if build . antcommands :
cmd + = build . antcommands
2012-02-21 01:01:07 +01:00
else :
2013-10-16 23:17:51 +02:00
cmd + = [ ' release ' ]
2013-11-04 20:19:31 +01:00
p = FDroidPopen ( cmd , cwd = root_dir )
2013-10-16 23:34:51 +02:00
2013-11-09 12:21:43 +01:00
bindir = os . path . join ( root_dir , ' bin ' )
2020-07-23 13:03:51 +02:00
if os . path . isdir ( os . path . join ( build_dir , ' .git ' ) ) :
import git
commit_id = common . get_head_commit_id ( git . repo . Repo ( build_dir ) )
else :
commit_id = build . commit
2014-01-28 22:14:53 +01:00
if p is not None and p . returncode != 0 :
2020-07-23 13:03:51 +02:00
raise BuildException ( " Build failed for %s : %s @ %s " % ( app . id , build . versionName , commit_id ) ,
p . output )
logging . info ( " Successfully built version {versionName} of {appid} from {commit_id} "
. format ( versionName = build . versionName , appid = app . id , commit_id = commit_id ) )
2012-02-21 01:01:07 +01:00
2016-02-15 13:01:38 +01:00
omethod = build . output_method ( )
if omethod == ' maven ' :
2013-11-04 20:19:31 +01:00
stdout_apk = ' \n ' . join ( [
2014-08-12 12:56:15 +02:00
line for line in p . output . splitlines ( ) if any (
a in line for a in ( ' .apk ' , ' .ap_ ' , ' .jar ' ) ) ] )
2012-06-28 13:52:35 +02:00
m = re . match ( r " .*^ \ [INFO \ ] .*apkbuilder.*/([^/]*) \ .apk " ,
2014-05-06 19:50:52 +02:00
stdout_apk , re . S | re . M )
2012-06-28 13:52:35 +02:00
if not m :
2013-10-23 19:53:07 +02:00
m = re . match ( r " .*^ \ [INFO \ ] Creating additional unsigned apk file .*/([^/]+) \ .apk[^l] " ,
2014-05-06 19:50:52 +02:00
stdout_apk , re . S | re . M )
2012-08-13 11:16:25 +02:00
if not m :
2013-11-08 18:56:28 +01:00
m = re . match ( r ' .*^ \ [INFO \ ] [^$]*aapt \ [package,[^$]* ' + bindir + r ' /([^/]+) \ .ap[_k][, \ ]] ' ,
2014-05-06 19:50:52 +02:00
stdout_apk , re . S | re . M )
2014-08-12 12:56:15 +02:00
if not m :
m = re . match ( r " .*^ \ [INFO \ ] Building jar: .*/ " + bindir + r " /(.+) \ .jar " ,
stdout_apk , re . S | re . M )
2012-06-28 13:52:35 +02:00
if not m :
2012-06-25 10:22:11 +02:00
raise BuildException ( ' Failed to find output ' )
2012-06-28 13:52:35 +02:00
src = m . group ( 1 )
2012-02-21 01:01:07 +01:00
src = os . path . join ( bindir , src ) + ' .apk '
2017-06-20 14:16:31 +02:00
2016-02-15 13:01:38 +01:00
elif omethod == ' gradle ' :
2015-12-24 16:41:39 +01:00
src = None
2017-08-26 12:55:34 +02:00
apk_dirs = [
2017-08-26 17:03:13 +02:00
# gradle plugin >= 3.0
2017-08-26 12:55:34 +02:00
os . path . join ( root_dir , ' build ' , ' outputs ' , ' apk ' , ' release ' ) ,
2017-08-26 17:03:13 +02:00
# gradle plugin < 3.0 and >= 0.11
2017-08-26 12:55:34 +02:00
os . path . join ( root_dir , ' build ' , ' outputs ' , ' apk ' ) ,
2017-08-26 17:03:13 +02:00
# really old path
2017-08-26 12:55:34 +02:00
os . path . join ( root_dir , ' build ' , ' apk ' ) ,
]
2021-01-29 15:28:55 +01:00
# If we build with gradle flavours with gradle plugin >= 3.0 the APK will be in
2017-08-26 17:03:13 +02:00
# a subdirectory corresponding to the flavour command used, but with different
# capitalization.
2017-08-26 12:55:34 +02:00
if flavours_cmd :
apk_dirs . append ( os . path . join ( root_dir , ' build ' , ' outputs ' , ' apk ' , transform_first_char ( flavours_cmd , str . lower ) , ' release ' ) )
for apks_dir in apk_dirs :
2015-12-31 09:37:08 +01:00
for apkglob in [ ' *-release-unsigned.apk ' , ' *-unsigned.apk ' , ' *.apk ' ] :
apks = glob . glob ( os . path . join ( apks_dir , apkglob ) )
if len ( apks ) > 1 :
raise BuildException ( ' More than one resulting apks found in %s ' % apks_dir ,
' \n ' . join ( apks ) )
if len ( apks ) == 1 :
src = apks [ 0 ]
break
if src is not None :
2015-12-24 16:41:39 +01:00
break
if src is None :
raise BuildException ( ' Failed to find any output apks ' )
2014-07-09 11:08:11 +02:00
2016-02-15 13:01:38 +01:00
elif omethod == ' ant ' :
2013-11-04 20:19:31 +01:00
stdout_apk = ' \n ' . join ( [
2014-07-01 18:04:41 +02:00
line for line in p . output . splitlines ( ) if ' .apk ' in line ] )
2013-11-04 20:19:31 +01:00
src = re . match ( r " .*^.*Creating (.+) for release.*$.* " , stdout_apk ,
2014-05-06 19:50:52 +02:00
re . S | re . M ) . group ( 1 )
2012-02-21 01:01:07 +01:00
src = os . path . join ( bindir , src )
2016-02-15 13:01:38 +01:00
elif omethod == ' raw ' :
2017-03-16 09:23:28 +01:00
output_path = common . replace_build_vars ( build . output , build )
globpath = os . path . join ( root_dir , output_path )
2016-02-15 13:01:38 +01:00
apks = glob . glob ( globpath )
if len ( apks ) > 1 :
raise BuildException ( ' Multiple apks match %s ' % globpath , ' \n ' . join ( apks ) )
if len ( apks ) < 1 :
raise BuildException ( ' No apks match %s ' % globpath )
src = os . path . normpath ( apks [ 0 ] )
2012-02-21 01:01:07 +01:00
2023-02-16 13:11:26 +01:00
# Run a postbuild command if one is required...
if build . postbuild :
logging . info ( f " Running ' postbuild ' commands in { root_dir } " )
cmd = common . replace_config_vars ( " ; " . join ( build . postbuild ) , build )
# Substitute source library paths into commands...
for name , number , libpath in srclibpaths :
cmd = cmd . replace ( f " $$ { name } $$ " , str ( Path . cwd ( ) / libpath ) )
cmd = cmd . replace ( ' $$OUT$$ ' , str ( Path ( src ) . resolve ( ) ) )
p = FDroidPopen ( [ ' bash ' , ' -e ' , ' -u ' , ' -o ' , ' pipefail ' , ' -x ' , ' -c ' , cmd ] , cwd = root_dir )
if p . returncode != 0 :
raise BuildException ( " Error running postbuild command for "
f " { app . id } : { build . versionName } " , p . output )
2013-04-15 14:04:28 +02:00
# Make sure it's not debuggable...
2024-04-01 12:40:14 +02:00
if common . is_debuggable_or_testOnly ( src ) :
raise BuildException (
" %s : debuggable or testOnly set in AndroidManifest.xml " % src
)
2013-04-15 14:04:28 +02:00
2012-02-21 01:01:07 +01:00
# By way of a sanity check, make sure the version and version
2021-01-29 15:28:55 +01:00
# code in our new APK match what we expect...
2014-07-05 15:29:12 +02:00
logging . debug ( " Checking " + src )
2012-06-28 13:52:35 +02:00
if not os . path . exists ( src ) :
2021-01-29 15:28:55 +01:00
raise BuildException ( " Unsigned APK is not at expected location of " + src )
2013-08-08 13:00:02 +02:00
2016-10-31 19:53:55 +01:00
if common . get_file_extension ( src ) == ' apk ' :
vercode , version = get_metadata_from_apk ( app , build , src )
2018-05-21 22:42:02 +02:00
if version != build . versionName or vercode != build . versionCode :
2016-10-31 19:53:55 +01:00
raise BuildException ( ( " Unexpected version/version code in output; "
2022-09-14 03:45:24 +02:00
" APK: ' %s ' / ' %d ' , "
" Expected: ' %s ' / ' %d ' " )
% ( version , vercode , build . versionName ,
build . versionCode ) )
2020-06-15 20:03:19 +02:00
if ( options . scan_binary or config . get ( ' scan_binary ' ) ) and not options . skipscan :
if scanner . scan_binary ( src ) :
2021-06-17 14:46:51 +02:00
raise BuildException ( " Found blocklisted packages in final apk! " )
2014-10-24 23:20:42 +02:00
2021-01-29 15:28:55 +01:00
# Copy the unsigned APK to our destination directory for further
2012-02-21 01:01:07 +01:00
# processing (by publish.py)...
2021-06-19 07:30:56 +02:00
dest = os . path . join (
output_dir ,
common . get_release_filename (
app , build , common . get_file_extension ( src )
)
)
2012-02-21 01:01:07 +01:00
shutil . copyfile ( src , dest )
# Move the source tarball into the output directory...
2014-01-28 22:13:18 +01:00
if output_dir != tmp_dir and not options . notarball :
2013-10-19 12:18:48 +02:00
shutil . move ( os . path . join ( tmp_dir , tarname ) ,
2014-05-01 00:23:57 +02:00
os . path . join ( output_dir , tarname ) )
2012-02-21 01:01:07 +01:00
2017-03-21 23:51:15 +01:00
def trybuild ( app , build , build_dir , output_dir , log_dir , also_check_dir ,
srclib_dir , extlib_dir , tmp_dir , repo_dir , vcs , test ,
server , force , onserver , refresh ) :
2021-06-07 12:26:57 +02:00
""" Build a particular version of an application, if it needs building.
2012-02-26 15:09:25 +01:00
2021-05-17 09:56:51 +02:00
Parameters
- - - - - - - - - -
output_dir
2021-06-07 12:49:16 +02:00
The directory where the build output will go .
2021-05-17 09:56:51 +02:00
Usually this is the ' unsigned ' directory .
repo_dir
The repo directory - used for checking if the build is necessary .
also_check_dir
An additional location for checking if the build
is necessary ( usually the archive repo )
test
True if building in test mode , in which case the build will
always happen , even if the output already exists . In test mode , the
output directory should be a temporary location , not any of the real
ones .
Returns
- - - - - - -
Boolean
True if the build was done , False if it wasn ' t necessary.
2012-02-26 15:09:25 +01:00
"""
2016-10-31 16:51:34 +01:00
dest_file = common . get_release_filename ( app , build )
2013-10-19 12:18:48 +02:00
2016-10-31 16:51:34 +01:00
dest = os . path . join ( output_dir , dest_file )
dest_repo = os . path . join ( repo_dir , dest_file )
2012-02-26 15:09:25 +01:00
2013-10-24 10:42:51 +02:00
if not test :
if os . path . exists ( dest ) or os . path . exists ( dest_repo ) :
2013-05-09 21:09:17 +02:00
return False
2013-10-24 10:42:51 +02:00
if also_check_dir :
2016-10-31 16:51:34 +01:00
dest_also = os . path . join ( also_check_dir , dest_file )
2013-10-24 10:42:51 +02:00
if os . path . exists ( dest_also ) :
return False
2015-11-28 17:55:27 +01:00
if build . disable and not options . force :
2012-02-26 15:09:25 +01:00
return False
2014-06-30 16:34:26 +02:00
logging . info ( " Building version %s ( %s ) of %s " % (
2016-11-23 17:52:04 +01:00
build . versionName , build . versionCode , app . id ) )
2012-02-26 15:09:25 +01:00
2012-02-26 18:14:15 +01:00
if server :
2012-08-31 15:48:50 +02:00
# When using server mode, still keep a local cache of the repo, by
# grabbing the source now.
2017-12-26 00:20:17 +01:00
vcs . gotorevision ( build . commit , refresh )
2012-08-31 15:48:50 +02:00
2021-09-11 17:15:44 +02:00
# Initialise submodules if required
if build . submodules :
vcs . initsubmodules ( )
2017-03-21 23:51:15 +01:00
build_server ( app , build , vcs , build_dir , output_dir , log_dir , force )
2012-02-26 15:09:25 +01:00
else :
2017-03-21 23:51:15 +01:00
build_local ( app , build , vcs , build_dir , output_dir , log_dir , srclib_dir , extlib_dir , tmp_dir , force , onserver , refresh )
2012-02-26 15:09:25 +01:00
return True
2018-01-28 07:56:19 +01:00
def force_halt_build ( timeout ) :
2021-06-07 12:26:57 +02:00
""" Halt the currently running Vagrant VM, to be called from a Timer. """
2018-01-28 07:56:19 +01:00
logging . error ( _ ( ' Force halting build after {0} sec timeout! ' ) . format ( timeout ) )
2018-01-20 22:16:39 +01:00
timeout_event . set ( )
2023-02-02 14:58:02 +01:00
if ssh_channel :
ssh_channel . close ( )
2018-01-15 01:03:47 +01:00
vm = vmtools . get_build_vm ( ' builder ' )
2023-02-02 09:28:02 +01:00
vm . destroy ( )
2018-01-15 01:03:47 +01:00
2023-05-05 09:27:46 +02:00
def keep_when_not_allowed ( ) :
""" Control if APKs signed by keys not in AllowedAPKSigningKeys are removed. """
return (
( options is not None and options . keep_when_not_allowed )
or ( config is not None and config . get ( ' keep_when_not_allowed ' ) )
or common . default_config [ ' keep_when_not_allowed ' ]
)
2012-02-26 15:09:25 +01:00
def parse_commandline ( ) :
2021-06-07 12:26:57 +02:00
""" Parse the command line.
2021-06-07 12:49:16 +02:00
2021-06-07 12:26:57 +02:00
Returns
- - - - - - -
options
parser
"""
2019-01-29 11:23:45 +01:00
parser = argparse . ArgumentParser ( usage = " %(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]] " )
2015-09-12 08:42:50 +02:00
common . setup_global_opts ( parser )
2020-10-01 11:28:09 +02:00
parser . add_argument ( " appid " , nargs = ' * ' , help = _ ( " application ID with optional versionCode in the form APPID[:VERCODE] " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -l " , " --latest " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Build only the latest version of each package " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -s " , " --stop " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Make the build stop on exceptions " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -t " , " --test " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Test mode - put output in the tmp directory only, and always build, even if the output already exists. " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --server " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Use build server " ) )
2018-01-30 20:45:03 +01:00
parser . add_argument ( " --reset-server " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Reset and create a brand new build server, even if the existing one appears to be ok. " ) )
2019-01-29 11:23:45 +01:00
# this option is internal API for telling fdroid that
# it's running inside a buildserver vm.
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --on-server " , dest = " onserver " , action = " store_true " , default = False ,
2019-01-29 11:23:45 +01:00
help = argparse . SUPPRESS )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --skip-scan " , dest = " skipscan " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Skip scanning the source code for binaries and other problems " ) )
2020-06-15 20:03:19 +02:00
parser . add_argument ( " --scan-binary " , action = " store_true " , default = False ,
help = _ ( " Scan the resulting APK(s) for known non-free classes. " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --no-tarball " , dest = " notarball " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Don ' t create a source tarball, useful when testing a build " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --no-refresh " , dest = " refresh " , action = " store_false " , default = True ,
2017-09-13 18:03:57 +02:00
help = _ ( " Don ' t refresh the repository, useful when testing a build with no internet connection " ) )
2024-01-25 12:15:02 +01:00
parser . add_argument ( " -r " , " --refresh-scanner " , dest = " refresh_scanner " , action = " store_true " , default = False ,
help = _ ( " Refresh and cache scanner rules and signatures from the network " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -f " , " --force " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode. " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -a " , " --all " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Build all applications available " ) )
2023-05-05 09:27:46 +02:00
parser . add_argument ( " --keep-when-not-allowed " , default = False , action = " store_true " ,
help = argparse . SUPPRESS )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -w " , " --wiki " , default = False , action = " store_true " ,
2021-11-24 18:04:51 +01:00
help = argparse . SUPPRESS )
2016-09-12 12:55:48 +02:00
metadata . add_metadata_arguments ( parser )
2015-09-04 11:37:05 +02:00
options = parser . parse_args ( )
2016-09-12 12:55:48 +02:00
metadata . warnings_action = options . W
2012-02-26 15:09:25 +01:00
2014-03-05 00:36:32 +01:00
# Force --stop with --on-server to get correct exit code
2013-10-21 22:16:41 +02:00
if options . onserver :
options . stop = True
2012-03-09 16:17:16 +01:00
if options . force and not options . test :
2015-09-04 11:37:05 +02:00
parser . error ( " option %s : Force is only allowed in test mode " % " force " )
2012-03-09 16:17:16 +01:00
2015-09-04 11:37:05 +02:00
return options , parser
2012-02-26 15:09:25 +01:00
2016-11-15 21:55:06 +01:00
2012-03-07 07:46:56 +01:00
options = None
2013-11-01 12:10:57 +01:00
config = None
2018-01-22 14:00:16 +01:00
fdroidserverid = None
start_timestamp = time . gmtime ( )
2020-02-18 23:16:18 +01:00
status_output = None
2018-01-20 22:16:39 +01:00
timeout_event = threading . Event ( )
2012-02-26 15:09:25 +01:00
2014-05-01 00:23:57 +02:00
2012-02-26 15:09:25 +01:00
def main ( ) :
2018-01-22 14:00:16 +01:00
global options , config , buildserverid , fdroidserverid
2013-05-09 21:09:17 +02:00
2015-09-04 11:37:05 +02:00
options , parser = parse_commandline ( )
2015-08-04 16:09:44 +02:00
2015-08-05 15:18:22 +02:00
# The defaults for .fdroid.* metadata that is included in a git repo are
# different than for the standard metadata/ layout because expectations
# are different. In this case, the most common user will be the app
# developer working on the latest update of the app on their own machine.
2016-03-27 21:36:16 +02:00
local_metadata_files = common . get_local_metadata_files ( )
2015-08-05 15:18:22 +02:00
if len ( local_metadata_files ) == 1 : # there is local metadata in an app's source
config = dict ( common . default_config )
# `fdroid build` should build only the latest version by default since
# most of the time the user will be building the most recent update
if not options . all :
options . latest = True
elif len ( local_metadata_files ) > 1 :
2015-08-04 16:09:44 +02:00
raise FDroidException ( " Only one local metadata file allowed! Found: "
2015-08-05 15:18:22 +02:00
+ " " . join ( local_metadata_files ) )
else :
if not os . path . isdir ( ' metadata ' ) and len ( local_metadata_files ) == 0 :
raise FDroidException ( " No app metadata found, nothing to process! " )
if not options . appid and not options . all :
parser . error ( " option %s : If you really want to build all the apps, use --all " % " all " )
2013-12-22 21:31:35 +01:00
2022-09-30 04:44:14 +02:00
config = common . read_config ( opts = options )
2013-11-01 12:10:57 +01:00
2013-10-31 16:37:39 +01:00
if config [ ' build_server_always ' ] :
2012-09-03 12:48:47 +02:00
options . server = True
2018-01-30 20:45:03 +01:00
if options . reset_server and not options . server :
parser . error ( " option %s : Using --reset-server without --server makes no sense " % " reset-server " )
2012-02-26 15:09:25 +01:00
log_dir = ' logs '
if not os . path . isdir ( log_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating log directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( log_dir )
tmp_dir = ' tmp '
if not os . path . isdir ( tmp_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating temporary directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( tmp_dir )
if options . test :
output_dir = tmp_dir
2012-01-22 00:10:59 +01:00
else :
2012-02-26 15:09:25 +01:00
output_dir = ' unsigned '
if not os . path . isdir ( output_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating output directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( output_dir )
2018-07-12 23:52:46 +02:00
binaries_dir = os . path . join ( output_dir , ' binaries ' )
2012-02-26 15:09:25 +01:00
2013-10-31 16:37:39 +01:00
if config [ ' archive_older ' ] != 0 :
2013-05-09 21:09:17 +02:00
also_check_dir = ' archive '
else :
also_check_dir = None
2020-02-20 09:02:39 +01:00
if options . onserver :
status_output = dict ( ) # HACK dummy placeholder
else :
status_output = common . setup_status_output ( start_timestamp )
2020-02-18 23:16:18 +01:00
2012-02-26 15:09:25 +01:00
repo_dir = ' repo '
build_dir = ' build '
if not os . path . isdir ( build_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating build directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( build_dir )
2013-05-20 13:34:03 +02:00
srclib_dir = os . path . join ( build_dir , ' srclib ' )
2012-02-26 15:09:25 +01:00
extlib_dir = os . path . join ( build_dir , ' extlib ' )
2014-05-20 23:44:47 +02:00
# Read all app and srclib metadata
2016-11-07 21:47:53 +01:00
pkgs = common . read_pkg_args ( options . appid , True )
2021-03-16 09:22:55 +01:00
allapps = metadata . read_metadata ( pkgs , sort_by_time = True )
2015-09-04 11:37:05 +02:00
apps = common . read_app_args ( options . appid , allapps , True )
2016-11-07 21:47:53 +01:00
2016-03-14 23:23:19 +01:00
for appid , app in list ( apps . items ( ) ) :
eliminate app.builds everywhere, it should be app['Builds']
The .txt format was the last place where the lowercase "builds" was used,
this converts references everywhere to be "Builds". This makes it possible
to load metadata YAML files with any YAML parser, then have it possible to
use fdroidserver methods on that data, like metadata.write_metadata().
The test files in tests/metadata/dump/*.yaml were manually edited by cutting
the builds: block and putting it the sort order for Builds: so the contents
should be unchanged.
```
sed -i \
-e 's/app\.builds/app.get('Builds', \[\])/g' \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\])/app.get('Builds', \[\])/g" \
-e "s/app\.get('Builds', \[\])\.append/app\['Builds'\].append/g" \
-e "s/app\['builds'\]/app.get('Builds', [])/g" \
*/*.*
```
2020-12-09 16:01:21 +01:00
if ( app . get ( ' Disabled ' ) and not options . force ) or not app . get ( ' RepoType ' ) or not app . get ( ' Builds ' , [ ] ) :
2014-08-16 12:46:02 +02:00
del apps [ appid ]
2013-12-11 19:08:15 +01:00
2014-08-16 12:46:02 +02:00
if not apps :
2014-07-07 15:41:32 +02:00
raise FDroidException ( " No apps to process. " )
2013-12-11 19:08:15 +01:00
2017-11-28 10:39:35 +01:00
# make sure enough open files are allowed to process everything
2021-06-18 10:24:44 +02:00
try :
import resource # not available on Windows
soft , hard = resource . getrlimit ( resource . RLIMIT_NOFILE )
if len ( apps ) > soft :
try :
soft = len ( apps ) * 2
if soft > hard :
soft = hard
resource . setrlimit ( resource . RLIMIT_NOFILE , ( soft , hard ) )
logging . debug ( _ ( ' Set open file limit to {integer} ' )
. format ( integer = soft ) )
except ( OSError , ValueError ) as e :
logging . warning ( _ ( ' Setting open file limit failed: ' ) + str ( e ) )
except ImportError :
pass
2017-11-28 10:39:35 +01:00
2013-12-11 19:08:15 +01:00
if options . latest :
2016-01-04 17:43:22 +01:00
for app in apps . values ( ) :
eliminate app.builds everywhere, it should be app['Builds']
The .txt format was the last place where the lowercase "builds" was used,
this converts references everywhere to be "Builds". This makes it possible
to load metadata YAML files with any YAML parser, then have it possible to
use fdroidserver methods on that data, like metadata.write_metadata().
The test files in tests/metadata/dump/*.yaml were manually edited by cutting
the builds: block and putting it the sort order for Builds: so the contents
should be unchanged.
```
sed -i \
-e 's/app\.builds/app.get('Builds', \[\])/g' \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\])/app.get('Builds', \[\])/g" \
-e "s/app\.get('Builds', \[\])\.append/app\['Builds'\].append/g" \
-e "s/app\['builds'\]/app.get('Builds', [])/g" \
*/*.*
```
2020-12-09 16:01:21 +01:00
for build in reversed ( app . get ( ' Builds ' , [ ] ) ) :
2015-11-28 17:55:27 +01:00
if build . disable and not options . force :
2014-01-20 09:58:17 +01:00
continue
eliminate app.builds everywhere, it should be app['Builds']
The .txt format was the last place where the lowercase "builds" was used,
this converts references everywhere to be "Builds". This makes it possible
to load metadata YAML files with any YAML parser, then have it possible to
use fdroidserver methods on that data, like metadata.write_metadata().
The test files in tests/metadata/dump/*.yaml were manually edited by cutting
the builds: block and putting it the sort order for Builds: so the contents
should be unchanged.
```
sed -i \
-e 's/app\.builds/app.get('Builds', \[\])/g' \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\])/app.get('Builds', \[\])/g" \
-e "s/app\.get('Builds', \[\])\.append/app\['Builds'\].append/g" \
-e "s/app\['builds'\]/app.get('Builds', [])/g" \
*/*.*
```
2020-12-09 16:01:21 +01:00
app [ ' Builds ' ] = [ build ]
2014-01-20 09:58:17 +01:00
break
2012-02-26 15:09:25 +01:00
2022-10-12 12:07:40 +02:00
if not options . onserver :
common . write_running_status_json ( status_output )
2012-02-26 15:09:25 +01:00
# Build applications...
2020-02-19 20:18:18 +01:00
failed_builds = [ ]
2021-03-11 18:52:31 +01:00
build_succeeded_ids = [ ]
2020-02-19 20:18:18 +01:00
status_output [ ' failedBuilds ' ] = failed_builds
2021-03-11 18:52:31 +01:00
status_output [ ' successfulBuildIds ' ] = build_succeeded_ids
2021-03-07 06:37:20 +01:00
# Only build for 72 hours, then stop gracefully.
endtime = time . time ( ) + 72 * 60 * 60
2018-01-19 22:35:06 +01:00
max_build_time_reached = False
2016-01-04 17:02:28 +01:00
for appid , app in apps . items ( ) :
2010-11-11 01:03:39 +01:00
2013-05-31 08:49:39 +02:00
first = True
2013-04-23 14:29:00 +02:00
eliminate app.builds everywhere, it should be app['Builds']
The .txt format was the last place where the lowercase "builds" was used,
this converts references everywhere to be "Builds". This makes it possible
to load metadata YAML files with any YAML parser, then have it possible to
use fdroidserver methods on that data, like metadata.write_metadata().
The test files in tests/metadata/dump/*.yaml were manually edited by cutting
the builds: block and putting it the sort order for Builds: so the contents
should be unchanged.
```
sed -i \
-e 's/app\.builds/app.get('Builds', \[\])/g' \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\]) =/app\['Builds'] =/g" \
-e "s/app\.get(Builds, \[\])/app.get('Builds', \[\])/g" \
-e "s/app\.get('Builds', \[\])\.append/app\['Builds'\].append/g" \
-e "s/app\['builds'\]/app.get('Builds', [])/g" \
*/*.*
```
2020-12-09 16:01:21 +01:00
for build in app . get ( ' Builds ' , [ ] ) :
2018-01-19 22:35:06 +01:00
if time . time ( ) > endtime :
max_build_time_reached = True
break
2018-01-28 07:56:19 +01:00
# Enable watchdog timer (2 hours by default).
if build . timeout is None :
timeout = 7200
else :
2022-09-14 03:45:24 +02:00
timeout = build . timeout
2018-01-28 07:56:19 +01:00
if options . server and timeout > 0 :
logging . debug ( _ ( ' Setting {0} sec timeout for this build ' ) . format ( timeout ) )
timer = threading . Timer ( timeout , force_halt_build , [ timeout ] )
2018-02-07 20:46:29 +01:00
timeout_event . clear ( )
2018-01-15 01:03:47 +01:00
timer . start ( )
else :
timer = None
2017-03-21 23:51:15 +01:00
tools_version_log = ' '
2012-01-02 15:09:20 +01:00
try :
2013-05-31 08:49:39 +02:00
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first :
2016-11-07 21:47:53 +01:00
vcs , build_dir = common . setup_vcs ( app )
2013-05-31 08:49:39 +02:00
first = False
2022-10-12 19:10:50 +02:00
logging . debug ( " Checking %s : %s " % ( appid , build . versionCode ) )
2017-03-21 23:51:15 +01:00
if trybuild ( app , build , build_dir , output_dir , log_dir ,
2014-05-06 19:50:52 +02:00
also_check_dir , srclib_dir , extlib_dir ,
tmp_dir , repo_dir , vcs , options . test ,
options . server , options . force ,
2015-07-14 12:32:39 +02:00
options . onserver , options . refresh ) :
2017-03-21 23:51:15 +01:00
toolslog = os . path . join ( log_dir ,
common . get_toolsversion_logname ( app , build ) )
if not options . onserver and os . path . exists ( toolslog ) :
with open ( toolslog , ' r ' ) as f :
tools_version_log = ' ' . join ( f . readlines ( ) )
os . remove ( toolslog )
2015-01-31 16:36:26 +01:00
2023-02-04 13:26:24 +01:00
if url := build . binary or app . Binaries :
2015-01-31 16:36:26 +01:00
# This is an app where we build from source, and
2021-01-29 15:28:55 +01:00
# verify the APK contents against a developer's
2015-01-31 16:36:26 +01:00
# binary. We get that binary now, and save it
# alongside our built one in the 'unsigend'
# directory.
2018-07-12 23:52:46 +02:00
if not os . path . isdir ( binaries_dir ) :
os . makedirs ( binaries_dir )
logging . info ( " Created directory for storing "
2018-07-12 23:43:19 +02:00
" developer supplied reference "
2018-07-12 23:52:46 +02:00
" binaries: ' {path} ' "
. format ( path = binaries_dir ) )
2016-11-23 17:52:04 +01:00
url = url . replace ( ' % v ' , build . versionName )
url = url . replace ( ' %c ' , str ( build . versionCode ) )
2015-01-31 16:36:26 +01:00
logging . info ( " ...retrieving " + url )
2021-04-12 09:19:11 +02:00
of = re . sub ( r ' \ .apk$ ' , ' .binary.apk ' , common . get_release_filename ( app , build ) )
2018-07-12 23:52:46 +02:00
of = os . path . join ( binaries_dir , of )
2017-04-20 17:48:38 +02:00
try :
net . download_file ( url , local_filename = of )
except requests . exceptions . HTTPError as e :
2017-05-22 21:33:52 +02:00
raise FDroidException (
2018-06-27 01:05:45 +02:00
' Downloading Binaries from %s failed. ' % url ) from e
2015-01-31 16:36:26 +01:00
2018-06-12 16:18:21 +02:00
# Now we check whether the build can be verified to
2017-04-04 18:58:16 +02:00
# match the supplied binary or not. Should the
# comparison fail, we mark this build as a failure
# and remove everything from the unsigend folder.
with tempfile . TemporaryDirectory ( ) as tmpdir :
unsigned_apk = \
2017-04-22 12:04:32 +02:00
common . get_release_filename ( app , build )
unsigned_apk = \
os . path . join ( output_dir , unsigned_apk )
2017-04-04 18:58:16 +02:00
compare_result = \
2017-04-20 12:44:22 +02:00
common . verify_apks ( of , unsigned_apk , tmpdir )
2017-04-04 18:58:16 +02:00
if compare_result :
2021-04-14 09:59:13 +02:00
if options . test :
logging . warning ( _ ( ' Keeping failed build " {apkfilename} " ' )
. format ( apkfilename = unsigned_apk ) )
else :
logging . debug ( ' removing %s ' , unsigned_apk )
os . remove ( unsigned_apk )
2017-04-23 11:33:51 +02:00
logging . debug ( ' removing %s ' , of )
os . remove ( of )
2017-04-04 18:58:16 +02:00
compare_result = compare_result . split ( ' \n ' )
line_count = len ( compare_result )
compare_result = compare_result [ : 299 ]
if line_count > len ( compare_result ) :
line_difference = \
line_count - len ( compare_result )
compare_result . append ( ' %d more lines ... ' %
line_difference )
compare_result = ' \n ' . join ( compare_result )
raise FDroidException ( ' compared built binary '
' to supplied reference '
' binary but failed ' ,
compare_result )
else :
logging . info ( ' compared built binary to '
' supplied reference binary '
' successfully ' )
2023-04-18 14:38:58 +02:00
used_key = common . apk_signer_fingerprint ( of )
expected_keys = app [ ' AllowedAPKSigningKeys ' ]
if used_key is None :
logging . warn ( _ ( ' reference binary missing '
' signature ' ) )
elif len ( expected_keys ) == 0 :
logging . warn ( _ ( ' AllowedAPKSigningKeys missing '
' but reference binary supplied ' ) )
elif used_key not in expected_keys :
2023-05-05 09:27:46 +02:00
if options . test or keep_when_not_allowed ( ) :
2023-04-18 14:38:58 +02:00
logging . warning ( _ ( ' Keeping failed build " {apkfilename} " ' )
. format ( apkfilename = unsigned_apk ) )
else :
logging . debug ( ' removing %s ' , unsigned_apk )
os . remove ( unsigned_apk )
logging . debug ( ' removing %s ' , of )
os . remove ( of )
raise FDroidException ( ' supplied reference '
' binary signed with '
' {signer} instead of '
' with {expected} ' .
format ( signer = used_key ,
expected = expected_keys ) )
else :
logging . info ( _ ( ' supplied reference binary has '
' allowed signer {signer} ' ) .
format ( signer = used_key ) )
2021-03-11 18:52:31 +01:00
build_succeeded_ids . append ( [ app [ ' id ' ] , build . versionCode ] )
2017-04-04 18:58:16 +02:00
2022-10-13 08:29:31 +02:00
if not options . onserver :
common . write_running_status_json ( status_output )
2012-01-02 15:09:20 +01:00
except VCSException as vcse :
2014-07-02 15:30:05 +02:00
reason = str ( vcse ) . split ( ' \n ' , 1 ) [ 0 ] if options . verbose else str ( vcse )
logging . error ( " VCS error while building app %s : %s " % (
2014-08-16 12:46:02 +02:00
appid , reason ) )
2012-01-08 19:27:54 +01:00
if options . stop :
2023-10-10 14:14:20 +02:00
logging . debug ( " Error encountered, stopping by user request. " )
2018-11-14 14:27:32 +01:00
common . force_exit ( 1 )
2022-11-09 14:46:25 +01:00
failed_builds . append ( ( appid , build . versionCode ) )
2021-08-27 13:27:42 +02:00
common . deploy_build_log_with_rsync (
appid , build . versionCode , " " . join ( traceback . format_exc ( ) )
)
2022-10-13 08:29:31 +02:00
if not options . onserver :
common . write_running_status_json ( status_output )
2015-11-14 13:05:37 +01:00
except FDroidException as e :
2021-11-16 14:28:38 +01:00
tstamp = time . strftime ( " % Y- % m- %d % H: % M: % SZ " , time . gmtime ( ) )
2015-11-14 13:05:37 +01:00
with open ( os . path . join ( log_dir , appid + ' .log ' ) , ' a+ ' ) as f :
2016-11-07 15:36:16 +01:00
f . write ( ' \n \n ============================================================ \n ' )
f . write ( ' versionCode: %s \n versionName: %s \n commit: %s \n ' %
2016-11-23 17:52:04 +01:00
( build . versionCode , build . versionName , build . commit ) )
2016-11-07 15:36:16 +01:00
f . write ( ' Build completed at '
2021-11-16 14:28:38 +01:00
+ tstamp + ' \n ' )
2016-11-07 15:36:16 +01:00
f . write ( ' \n ' + tools_version_log + ' \n ' )
2015-11-14 13:05:37 +01:00
f . write ( str ( e ) )
logging . error ( " Could not build app %s : %s " % ( appid , e ) )
if options . stop :
2023-10-10 14:14:20 +02:00
logging . debug ( " Error encountered, stopping by user request. " )
2018-11-14 14:27:32 +01:00
common . force_exit ( 1 )
2022-11-09 14:46:25 +01:00
failed_builds . append ( ( appid , build . versionCode ) )
2021-08-27 13:27:42 +02:00
common . deploy_build_log_with_rsync (
appid , build . versionCode , " " . join ( traceback . format_exc ( ) )
)
2022-10-13 08:29:31 +02:00
if not options . onserver :
common . write_running_status_json ( status_output )
2022-11-09 14:46:25 +01:00
except Exception :
2014-07-02 15:30:05 +02:00
logging . error ( " Could not build app %s due to unknown error: %s " % (
2014-08-16 12:46:02 +02:00
appid , traceback . format_exc ( ) ) )
2012-01-08 19:27:54 +01:00
if options . stop :
2023-10-10 14:14:20 +02:00
logging . debug ( " Error encountered, stopping by user request. " )
2018-11-14 14:27:32 +01:00
common . force_exit ( 1 )
2022-11-09 14:46:25 +01:00
failed_builds . append ( ( appid , build . versionCode ) )
2021-08-27 13:27:42 +02:00
common . deploy_build_log_with_rsync (
appid , build . versionCode , " " . join ( traceback . format_exc ( ) )
)
2022-10-13 08:29:31 +02:00
if not options . onserver :
common . write_running_status_json ( status_output )
2013-05-03 08:55:40 +02:00
2018-01-15 01:03:47 +01:00
if timer :
timer . cancel ( ) # kill the watchdog timer
2018-01-19 22:35:06 +01:00
if max_build_time_reached :
2020-02-18 23:16:18 +01:00
status_output [ ' maxBuildTimeReached ' ] = True
2018-01-19 22:35:06 +01:00
logging . info ( " Stopping after global build timeout... " )
break
2022-10-12 15:33:27 +02:00
for app in build_succeeded_ids :
logging . info ( " success: %s " % app [ 0 ] )
2012-02-26 15:09:25 +01:00
2013-10-23 19:53:33 +02:00
if not options . verbose :
2020-02-19 20:18:18 +01:00
for fb in failed_builds :
2022-11-09 14:46:25 +01:00
logging . info ( ' Build for app {} : {} failed ' . format ( * fb ) )
2012-02-26 15:09:25 +01:00
2017-09-13 18:03:57 +02:00
logging . info ( _ ( " Finished " ) )
2022-10-12 15:33:27 +02:00
if len ( build_succeeded_ids ) > 0 :
2017-09-13 17:33:57 +02:00
logging . info ( ngettext ( " {} build succeeded " ,
2022-10-12 15:33:27 +02:00
" {} builds succeeded " , len ( build_succeeded_ids ) ) . format ( len ( build_succeeded_ids ) ) )
2020-02-19 20:18:18 +01:00
if len ( failed_builds ) > 0 :
2017-09-13 17:33:57 +02:00
logging . info ( ngettext ( " {} build failed " ,
2020-02-19 20:18:18 +01:00
" {} builds failed " , len ( failed_builds ) ) . format ( len ( failed_builds ) ) )
2011-01-04 22:44:14 +01:00
2021-11-24 18:02:01 +01:00
if options . server :
2018-01-22 14:00:16 +01:00
if os . cpu_count ( ) :
2021-03-04 09:50:48 +01:00
status_output [ ' hostOsCpuCount ' ] = os . cpu_count ( )
2018-01-22 14:00:16 +01:00
if os . path . isfile ( ' /proc/meminfo ' ) and os . access ( ' /proc/meminfo ' , os . R_OK ) :
with open ( ' /proc/meminfo ' ) as fp :
for line in fp :
m = re . search ( r ' MemTotal: \ s*([0-9].*) ' , line )
if m :
2021-03-04 09:50:48 +01:00
status_output [ ' hostProcMeminfoMemTotal ' ] = m . group ( 1 )
2018-01-22 14:00:16 +01:00
break
2022-11-15 08:50:23 +01:00
buildserver_config = ' builder/Vagrantfile '
2019-02-07 19:48:03 +01:00
if os . path . isfile ( buildserver_config ) and os . access ( buildserver_config , os . R_OK ) :
with open ( buildserver_config ) as configfile :
for line in configfile :
m = re . search ( r ' cpus \ s*= \ s*([0-9].*) ' , line )
if m :
2021-03-04 09:50:48 +01:00
status_output [ ' guestVagrantVmCpus ' ] = m . group ( 1 )
2019-02-07 19:48:03 +01:00
m = re . search ( r ' memory \ s*= \ s*([0-9].*) ' , line )
if m :
2021-03-04 09:50:48 +01:00
status_output [ ' guestVagrantVmMemory ' ] = m . group ( 1 )
2018-01-22 14:00:16 +01:00
2021-03-04 09:50:48 +01:00
if buildserverid :
status_output [ ' buildserver ' ] = { ' commitId ' : buildserverid }
2020-02-20 09:02:39 +01:00
if not options . onserver :
common . write_status_json ( status_output )
2020-02-18 23:16:18 +01:00
2017-12-02 13:24:13 +01:00
# hack to ensure this exits, even is some threads are still running
2018-11-14 14:27:32 +01:00
common . force_exit ( )
2012-01-03 17:02:28 +01:00
2016-11-15 21:55:06 +01:00
2012-02-26 15:09:25 +01:00
if __name__ == " __main__ " :
main ( )