2016-01-04 16:33:20 +01:00
#!/usr/bin/env python3
2010-11-11 01:03:39 +01:00
#
# build.py - part of the FDroid server tools
2014-05-02 00:04:51 +02:00
# Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
2014-01-28 14:07:19 +01:00
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
2010-11-11 01:03:39 +01:00
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
2014-07-09 11:08:11 +02:00
import glob
2010-11-11 01:03:39 +01:00
import subprocess
2018-08-29 15:43:16 +02:00
import posixpath
2010-11-11 01:03:39 +01:00
import re
2017-11-28 10:39:35 +01:00
import resource
2018-01-22 14:00:16 +01:00
import sys
2011-01-03 00:26:12 +01:00
import tarfile
2018-01-15 01:03:47 +01:00
import threading
2012-02-03 17:01:35 +01:00
import traceback
2013-05-15 19:18:24 +02:00
import time
2017-04-20 17:48:38 +02:00
import requests
2017-04-04 18:58:16 +02:00
import tempfile
2019-01-29 11:23:45 +01:00
import argparse
2016-01-04 17:43:13 +01:00
from configparser import ConfigParser
2014-01-27 16:45:30 +01:00
import logging
2017-09-13 17:33:57 +02:00
from gettext import ngettext
2010-11-11 01:03:39 +01:00
2017-09-13 18:03:57 +02:00
from . import _
2016-01-04 17:37:35 +01:00
from . import common
from . import net
from . import metadata
from . import scanner
2017-03-26 01:51:28 +01:00
from . import vmtools
2018-09-17 22:44:53 +02:00
from . common import FDroidPopen
2017-05-22 21:33:52 +02:00
from . exception import FDroidException , BuildException , VCSException
2012-02-21 01:01:07 +01:00
2014-05-02 00:04:51 +02:00
try :
import paramiko
2014-05-02 04:46:51 +02:00
except ImportError :
pass
2014-05-01 00:23:57 +02:00
2014-05-02 05:39:33 +02:00
2014-05-02 00:04:51 +02:00
# Note that 'force' here also implies test mode.
2017-03-21 23:51:15 +01:00
def build_server ( app , build , vcs , build_dir , output_dir , log_dir , force ) :
""" Do a build on the builder vm.
: param app : app metadata dict
: param build :
: param vcs : version control system controller object
: param build_dir : local source - code checkout of app
: param output_dir : target folder for the build result
: param force :
"""
2014-05-02 00:04:51 +02:00
2017-02-09 23:49:42 +01:00
global buildserverid
2014-05-02 04:46:51 +02:00
try :
paramiko
2018-06-27 01:05:45 +02:00
except NameError as e :
raise BuildException ( " Paramiko is required to use the buildserver " ) from e
2014-05-02 00:04:51 +02:00
if options . verbose :
2015-11-21 02:24:28 +01:00
logging . getLogger ( " paramiko " ) . setLevel ( logging . INFO )
2014-05-02 00:04:51 +02:00
else :
logging . getLogger ( " paramiko " ) . setLevel ( logging . WARN )
2018-01-30 20:45:03 +01:00
sshinfo = vmtools . get_clean_builder ( ' builder ' , options . reset_server )
2014-05-02 00:04:51 +02:00
2018-06-12 16:18:21 +02:00
output = None
2014-05-02 00:04:51 +02:00
try :
2017-02-09 23:49:42 +01:00
if not buildserverid :
2018-06-27 01:05:45 +02:00
try :
buildserverid = subprocess . check_output ( [ ' vagrant ' , ' ssh ' , ' -c ' ,
' cat /home/vagrant/buildserverid ' ] ,
cwd = ' builder ' ) . strip ( ) . decode ( )
2020-02-18 23:16:18 +01:00
status_output [ ' buildserverid ' ] = buildserverid
2018-06-27 01:05:45 +02:00
logging . debug ( _ ( ' Fetched buildserverid from VM: {buildserverid} ' )
. format ( buildserverid = buildserverid ) )
except Exception as e :
if type ( buildserverid ) is not str or not re . match ( ' ^[0-9a-f] {40} $ ' , buildserverid ) :
logging . info ( subprocess . check_output ( [ ' vagrant ' , ' status ' ] , cwd = " builder " ) )
raise FDroidException ( " Could not obtain buildserverid from buldserver VM. "
" (stored inside the buildserver VM at ' /home/vagrant/buildserverid ' ) "
" Please reset your buildserver, the setup VM is broken. " ) from e
2012-09-06 19:36:34 +02:00
# Open SSH connection...
2014-01-27 16:45:30 +01:00
logging . info ( " Connecting to virtual machine... " )
2014-04-03 18:14:59 +02:00
sshs = paramiko . SSHClient ( )
sshs . set_missing_host_key_policy ( paramiko . AutoAddPolicy ( ) )
2014-05-02 00:04:51 +02:00
sshs . connect ( sshinfo [ ' hostname ' ] , username = sshinfo [ ' user ' ] ,
port = sshinfo [ ' port ' ] , timeout = 300 ,
look_for_keys = False , key_filename = sshinfo [ ' idfile ' ] )
2012-09-06 19:36:34 +02:00
2018-08-29 15:43:16 +02:00
homedir = posixpath . join ( ' /home ' , sshinfo [ ' user ' ] )
2014-05-02 00:24:54 +02:00
2012-09-06 19:36:34 +02:00
# Get an SFTP connection...
ftp = sshs . open_sftp ( )
2016-07-01 14:00:38 +02:00
ftp . get_channel ( ) . settimeout ( 60 )
2012-09-06 19:36:34 +02:00
# Put all the necessary files in place...
2014-05-02 00:24:54 +02:00
ftp . chdir ( homedir )
2012-09-06 19:36:34 +02:00
# Helper to copy the contents of a directory to the server...
def send_dir ( path ) :
2017-11-24 22:41:06 +01:00
logging . debug ( " rsyncing " + path + " to " + ftp . getcwd ( ) )
2017-12-06 09:47:08 +01:00
# TODO this should move to `vagrant rsync` from >= v1.5
2017-12-05 21:31:55 +01:00
try :
2018-05-29 12:31:56 +02:00
subprocess . check_output ( [ ' rsync ' , ' --recursive ' , ' --perms ' , ' --links ' , ' --quiet ' , ' --rsh= '
+ ' ssh -o StrictHostKeyChecking=no '
+ ' -o UserKnownHostsFile=/dev/null '
+ ' -o LogLevel=FATAL '
+ ' -o IdentitiesOnly=yes '
+ ' -o PasswordAuthentication=no '
+ ' -p ' + str ( sshinfo [ ' port ' ] )
+ ' -i ' + sshinfo [ ' idfile ' ] ,
2017-12-05 21:31:55 +01:00
path ,
2017-12-05 21:37:04 +01:00
sshinfo [ ' user ' ] + " @ " + sshinfo [ ' hostname ' ] + " : " + ftp . getcwd ( ) ] ,
2017-12-05 21:31:55 +01:00
stderr = subprocess . STDOUT )
except subprocess . CalledProcessError as e :
raise FDroidException ( str ( e ) , e . output . decode ( ) )
2012-09-06 19:36:34 +02:00
2014-01-27 16:45:30 +01:00
logging . info ( " Preparing server for build... " )
2012-09-06 19:36:34 +02:00
serverpath = os . path . abspath ( os . path . dirname ( __file__ ) )
2015-09-24 11:19:17 +02:00
ftp . mkdir ( ' fdroidserver ' )
ftp . chdir ( ' fdroidserver ' )
ftp . put ( os . path . join ( serverpath , ' .. ' , ' fdroid ' ) , ' fdroid ' )
2018-07-08 17:34:17 +02:00
ftp . put ( os . path . join ( serverpath , ' .. ' , ' gradlew-fdroid ' ) , ' gradlew-fdroid ' )
2018-08-29 14:54:37 +02:00
ftp . chmod ( ' fdroid ' , 0o755 ) # nosec B103 permissions are appropriate
ftp . chmod ( ' gradlew-fdroid ' , 0o755 ) # nosec B103 permissions are appropriate
2015-09-24 11:19:17 +02:00
send_dir ( os . path . join ( serverpath ) )
ftp . chdir ( homedir )
2014-02-17 19:16:14 +01:00
ftp . put ( os . path . join ( serverpath , ' .. ' , ' buildserver ' ,
2014-12-31 16:42:26 +01:00
' config.buildserver.py ' ) , ' config.py ' )
2013-11-12 11:55:27 +01:00
ftp . chmod ( ' config.py ' , 0o600 )
2012-09-06 19:36:34 +02:00
2014-04-10 14:58:42 +02:00
# Copy over the ID (head commit hash) of the fdroidserver in use...
2018-01-23 23:56:15 +01:00
with open ( os . path . join ( os . getcwd ( ) , ' tmp ' , ' fdroidserverid ' ) , ' wb ' ) as fp :
fp . write ( subprocess . check_output ( [ ' git ' , ' rev-parse ' , ' HEAD ' ] ,
cwd = serverpath ) )
2014-04-10 14:58:42 +02:00
ftp . put ( ' tmp/fdroidserverid ' , ' fdroidserverid ' )
2012-09-06 19:36:34 +02:00
# Copy the metadata - just the file for this app...
ftp . mkdir ( ' metadata ' )
2013-05-22 11:17:02 +02:00
ftp . mkdir ( ' srclibs ' )
2012-09-06 19:36:34 +02:00
ftp . chdir ( ' metadata ' )
2017-05-16 16:28:24 +02:00
ftp . put ( app . metadatapath , os . path . basename ( app . metadatapath ) )
2012-09-08 10:56:20 +02:00
# And patches if there are any...
2015-11-28 13:09:47 +01:00
if os . path . exists ( os . path . join ( ' metadata ' , app . id ) ) :
send_dir ( os . path . join ( ' metadata ' , app . id ) )
2013-05-22 11:17:02 +02:00
2014-05-02 00:24:54 +02:00
ftp . chdir ( homedir )
2012-09-06 19:36:34 +02:00
# Create the build directory...
ftp . mkdir ( ' build ' )
2012-08-31 15:48:50 +02:00
ftp . chdir ( ' build ' )
2012-09-06 19:36:34 +02:00
ftp . mkdir ( ' extlib ' )
2013-05-20 22:00:31 +02:00
ftp . mkdir ( ' srclib ' )
2012-09-06 19:36:34 +02:00
# Copy any extlibs that are required...
2015-11-28 17:55:27 +01:00
if build . extlibs :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' build ' , ' extlib ' ) )
2015-11-28 17:55:27 +01:00
for lib in build . extlibs :
2013-09-11 13:45:02 +02:00
lib = lib . strip ( )
2013-11-12 21:14:16 +01:00
libsrc = os . path . join ( ' build/extlib ' , lib )
if not os . path . exists ( libsrc ) :
raise BuildException ( " Missing extlib {0} " . format ( libsrc ) )
2012-09-06 19:36:34 +02:00
lp = lib . split ( ' / ' )
for d in lp [ : - 1 ] :
2013-03-05 20:47:52 +01:00
if d not in ftp . listdir ( ) :
ftp . mkdir ( d )
2012-09-06 19:36:34 +02:00
ftp . chdir ( d )
2013-11-12 21:14:16 +01:00
ftp . put ( libsrc , lp [ - 1 ] )
2017-09-13 18:03:57 +02:00
for _ignored in lp [ : - 1 ] :
2012-09-06 19:36:34 +02:00
ftp . chdir ( ' .. ' )
# Copy any srclibs that are required...
2012-09-08 10:56:20 +02:00
srclibpaths = [ ]
2015-11-28 17:55:27 +01:00
if build . srclibs :
for lib in build . srclibs :
2014-05-01 00:23:57 +02:00
srclibpaths . append (
2015-06-03 13:51:41 +02:00
common . getsrclib ( lib , ' build/srclib ' , basepath = True , prepare = False ) )
2013-11-18 10:24:02 +01:00
2012-09-08 10:56:20 +02:00
# If one was used for the main source, add that too.
basesrclib = vcs . getsrclib ( )
if basesrclib :
srclibpaths . append ( basesrclib )
2013-11-16 22:49:37 +01:00
for name , number , lib in srclibpaths :
2014-01-27 16:45:30 +01:00
logging . info ( " Sending srclib ' %s ' " % lib )
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' build ' , ' srclib ' ) )
2012-09-20 15:16:08 +02:00
if not os . path . exists ( lib ) :
raise BuildException ( " Missing srclib directory ' " + lib + " ' " )
2013-07-18 22:56:37 +02:00
fv = ' .fdroidvcs- ' + name
ftp . put ( os . path . join ( ' build/srclib ' , fv ) , fv )
2012-09-08 10:56:20 +02:00
send_dir ( lib )
2013-05-22 11:17:02 +02:00
# Copy the metadata file too...
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' srclibs ' ) )
2020-08-24 16:23:13 +02:00
srclibsfile = os . path . join ( ' srclibs ' , name + ' .yml ' )
if os . path . isfile ( srclibsfile ) :
ftp . put ( srclibsfile , os . path . basename ( srclibsfile ) )
2020-04-16 11:32:37 +02:00
else :
2020-08-24 16:23:13 +02:00
raise BuildException ( _ ( ' cannot find required srclibs: " {path} " ' )
. format ( path = srclibsfile ) )
2013-07-18 22:56:37 +02:00
# Copy the main app source code
# (no need if it's a srclib)
if ( not basesrclib ) and os . path . exists ( build_dir ) :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' build ' ) )
2015-11-28 13:09:47 +01:00
fv = ' .fdroidvcs- ' + app . id
2013-07-18 22:56:37 +02:00
ftp . put ( os . path . join ( ' build ' , fv ) , fv )
send_dir ( build_dir )
2012-09-06 19:36:34 +02:00
# Execute the build script...
2014-01-27 16:45:30 +01:00
logging . info ( " Starting build... " )
2012-09-06 19:36:34 +02:00
chan = sshs . get_transport ( ) . open_session ( )
2014-01-16 11:17:22 +01:00
chan . get_pty ( )
2018-08-29 15:43:16 +02:00
cmdline = posixpath . join ( homedir , ' fdroidserver ' , ' fdroid ' )
2015-09-24 11:19:17 +02:00
cmdline + = ' build --on-server '
2012-09-27 20:58:03 +02:00
if force :
cmdline + = ' --force --test '
2013-11-24 11:29:09 +01:00
if options . verbose :
cmdline + = ' --verbose '
2017-03-01 17:27:17 +01:00
if options . skipscan :
cmdline + = ' --skip-scan '
2018-07-09 14:48:59 +02:00
if options . notarball :
cmdline + = ' --no-tarball '
2016-11-23 17:52:04 +01:00
cmdline + = " %s : %s " % ( app . id , build . versionCode )
2018-08-29 17:20:01 +02:00
chan . exec_command ( ' bash --login -c " ' + cmdline + ' " ' ) # nosec B601 inputs are sanitized
2017-02-09 23:48:40 +01:00
2017-09-14 01:38:34 +02:00
# Fetch build process output ...
try :
cmd_stdout = chan . makefile ( ' rb ' , 1024 )
output = bytes ( )
2018-01-17 17:13:12 +01:00
output + = common . get_android_tools_version_log ( build . ndk_path ( ) ) . encode ( )
2017-09-14 01:38:34 +02:00
while not chan . exit_status_ready ( ) :
line = cmd_stdout . readline ( )
if line :
if options . verbose :
logging . debug ( " buildserver > " + str ( line , ' utf-8 ' ) . rstrip ( ) )
output + = line
else :
time . sleep ( 0.05 )
for line in cmd_stdout . readlines ( ) :
if options . verbose :
logging . debug ( " buildserver > " + str ( line , ' utf-8 ' ) . rstrip ( ) )
output + = line
finally :
cmd_stdout . close ( )
# Check build process exit status ...
2014-01-27 16:45:30 +01:00
logging . info ( " ...getting exit status " )
2012-09-06 19:36:34 +02:00
returncode = chan . recv_exit_status ( )
if returncode != 0 :
2018-01-20 22:16:39 +01:00
if timeout_event . is_set ( ) :
message = " Timeout exceeded! Build VM force-stopped for {0} : {1} "
else :
message = " Build.py failed on server for {0} : {1} "
raise BuildException ( message . format ( app . id , build . versionName ) ,
None if options . verbose else str ( output , ' utf-8 ' ) )
2012-09-06 19:36:34 +02:00
2017-03-21 23:51:15 +01:00
# Retreive logs...
toolsversion_log = common . get_toolsversion_logname ( app , build )
try :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , log_dir ) )
2017-03-21 23:51:15 +01:00
ftp . get ( toolsversion_log , os . path . join ( log_dir , toolsversion_log ) )
logging . debug ( ' retrieved %s ' , toolsversion_log )
2017-03-22 14:01:32 +01:00
except Exception as e :
2020-05-25 17:36:58 +02:00
logging . warning ( ' could not get %s from builder vm: %s ' % ( toolsversion_log , e ) )
2017-03-21 23:51:15 +01:00
2012-09-06 19:36:34 +02:00
# Retrieve the built files...
2014-01-27 16:45:30 +01:00
logging . info ( " Retrieving build output... " )
2013-01-22 10:49:33 +01:00
if force :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' tmp ' ) )
2012-09-27 20:58:03 +02:00
else :
2018-08-29 15:43:16 +02:00
ftp . chdir ( posixpath . join ( homedir , ' unsigned ' ) )
2016-10-31 16:51:34 +01:00
apkfile = common . get_release_filename ( app , build )
2015-11-28 17:55:27 +01:00
tarball = common . getsrcname ( app , build )
2012-09-06 19:36:34 +02:00
try :
ftp . get ( apkfile , os . path . join ( output_dir , apkfile ) )
2014-01-28 22:13:18 +01:00
if not options . notarball :
ftp . get ( tarball , os . path . join ( output_dir , tarball ) )
2017-03-22 14:01:32 +01:00
except Exception :
2014-05-02 00:04:51 +02:00
raise BuildException (
2017-09-14 01:38:34 +02:00
" Build failed for {0} : {1} - missing output files " . format (
app . id , build . versionName ) , None if options . verbose else str ( output , ' utf-8 ' ) )
2012-09-06 19:36:34 +02:00
ftp . close ( )
finally :
# Suspend the build server.
2017-04-14 15:58:16 +02:00
vm = vmtools . get_build_vm ( ' builder ' )
vm . suspend ( )
2012-02-21 01:01:07 +01:00
2018-06-12 16:18:21 +02:00
# deploy logfile to repository web server
if output :
2018-06-23 23:26:42 +02:00
common . deploy_build_log_with_rsync ( app . id , build . versionCode , output )
2018-06-12 16:18:21 +02:00
else :
logging . debug ( ' skip publishing full build logs: '
' no output present ' )
2014-05-01 00:23:57 +02:00
2016-06-20 20:00:59 +02:00
def force_gradle_build_tools ( build_dir , build_tools ) :
2014-01-11 13:05:02 +01:00
for root , dirs , files in os . walk ( build_dir ) :
2015-04-06 17:42:26 +02:00
for filename in files :
if not filename . endswith ( ' .gradle ' ) :
continue
2015-03-31 23:08:32 +02:00
path = os . path . join ( root , filename )
2015-04-17 13:02:47 +02:00
if not os . path . isfile ( path ) :
continue
2016-06-20 20:00:59 +02:00
logging . debug ( " Forcing build-tools %s in %s " % ( build_tools , path ) )
2015-10-14 19:34:30 +02:00
common . regsub_file ( r """ ( \ s*)buildToolsVersion([ \ s=]+).* """ ,
2016-06-20 20:00:59 +02:00
r """ \ 1buildToolsVersion \ 2 ' %s ' """ % build_tools ,
2015-07-30 22:13:12 +02:00
path )
2013-08-22 18:28:30 +02:00
2012-02-21 01:01:07 +01:00
2017-08-26 12:55:34 +02:00
def transform_first_char ( string , method ) :
""" Uses method() on the first character of string. """
2015-01-02 22:26:22 +01:00
if len ( string ) == 0 :
return string
if len ( string ) == 1 :
2017-08-26 12:55:34 +02:00
return method ( string )
return method ( string [ 0 ] ) + string [ 1 : ]
2015-01-02 22:26:22 +01:00
2020-02-19 20:18:18 +01:00
def add_failed_builds_entry ( failed_builds , appid , build , entry ) :
failed_builds . append ( [ appid , int ( build . versionCode ) , str ( entry ) ] )
2017-04-13 14:18:48 +02:00
def get_metadata_from_apk ( app , build , apkfile ) :
2018-09-17 22:44:53 +02:00
""" get the required metadata from the built APK
2017-04-13 14:18:48 +02:00
2018-09-17 22:44:53 +02:00
versionName is allowed to be a blank string , i . e . ' '
"""
2017-04-13 14:18:48 +02:00
2018-09-17 22:44:53 +02:00
appid , versionCode , versionName = common . get_apk_id ( apkfile )
native_code = common . get_native_code ( apkfile )
2016-10-31 19:53:55 +01:00
2018-09-17 22:44:53 +02:00
if build . buildjni and build . buildjni != [ ' no ' ] and not native_code :
raise BuildException ( " Native code should have been built but none was packaged " )
2016-10-31 19:53:55 +01:00
if build . novcheck :
2018-09-17 22:44:53 +02:00
versionCode = build . versionCode
versionName = build . versionName
if not versionCode or versionName is None :
2016-10-31 19:53:55 +01:00
raise BuildException ( " Could not find version information in build in output " )
2018-09-17 22:44:53 +02:00
if not appid :
2016-10-31 19:53:55 +01:00
raise BuildException ( " Could not find package ID in output " )
2018-09-17 22:44:53 +02:00
if appid != app . id :
raise BuildException ( " Wrong package ID - build " + appid + " but expected " + app . id )
2016-10-31 19:53:55 +01:00
2018-09-17 22:44:53 +02:00
return versionCode , versionName
2016-10-31 19:53:55 +01:00
2017-03-21 23:51:15 +01:00
def build_local ( app , build , vcs , build_dir , output_dir , log_dir , srclib_dir , extlib_dir , tmp_dir , force , onserver , refresh ) :
2012-02-26 15:09:25 +01:00
""" Do a build locally. """
2015-11-28 17:55:27 +01:00
ndk_path = build . ndk_path ( )
2016-07-13 17:32:52 +02:00
if build . ndk or ( build . buildjni and build . buildjni != [ ' no ' ] ) :
2015-11-28 17:55:27 +01:00
if not ndk_path :
2016-08-02 11:09:32 +02:00
logging . critical ( " Android NDK version ' %s ' could not be found! " % build . ndk or ' r12b ' )
2015-01-03 00:02:54 +01:00
logging . critical ( " Configured versions: " )
2016-01-04 17:02:28 +01:00
for k , v in config [ ' ndk_paths ' ] . items ( ) :
2015-01-03 00:02:54 +01:00
if k . endswith ( " _orig " ) :
continue
logging . critical ( " %s : %s " % ( k , v ) )
2017-05-22 21:33:52 +02:00
raise FDroidException ( )
2015-11-28 17:55:27 +01:00
elif not os . path . isdir ( ndk_path ) :
logging . critical ( " Android NDK ' %s ' is not a directory! " % ndk_path )
2017-05-22 21:33:52 +02:00
raise FDroidException ( )
2014-02-17 20:03:55 +01:00
2015-08-05 14:39:58 +02:00
common . set_FDroidPopen_env ( build )
2015-01-06 19:41:55 +01:00
2017-03-21 23:51:15 +01:00
# create ..._toolsversion.log when running in builder vm
if onserver :
2017-06-28 23:01:45 +02:00
# before doing anything, run the sudo commands to setup the VM
if build . sudo :
logging . info ( " Running ' sudo ' commands in %s " % os . getcwd ( ) )
2020-02-11 19:44:06 +01:00
p = FDroidPopen ( [ ' sudo ' , ' DEBIAN_FRONTEND=noninteractive ' ,
2020-01-14 23:20:48 +01:00
' bash ' , ' -x ' , ' -c ' , build . sudo ] )
2017-06-28 23:01:45 +02:00
if p . returncode != 0 :
raise BuildException ( " Error running sudo command for %s : %s " %
( app . id , build . versionName ) , p . output )
build: force purging of sudo, ignore error message
Fixes bb758d3f, spotted by @bubu:
DEBUG: buildserver > DEBUG: > sudo apt-get -y purge sudo
DEBUG: buildserver > Reading package lists...
DEBUG: buildserver > Building dependency tree...
DEBUG: buildserver > Reading state information...
DEBUG: buildserver > The following packages will be REMOVED:
DEBUG: buildserver > sudo*
DEBUG: buildserver > 0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded.
DEBUG: buildserver > After this operation, 2,391 kB disk space will be freed.
(Reading database ... 68491 files and directories currently installed.)
DEBUG: buildserver > Removing sudo (1.8.10p3-1+deb8u4) ...
DEBUG: buildserver > You have asked that the sudo package be removed,
DEBUG: buildserver > but no root password has been set.
DEBUG: buildserver > Without sudo, you may not be able to gain administrative privileges.
DEBUG: buildserver >
DEBUG: buildserver > If you would prefer to access the root account with su(1)
DEBUG: buildserver > or by logging in directly,
DEBUG: buildserver > you must set a root password with "sudo passwd".
DEBUG: buildserver >
DEBUG: buildserver > If you have arranged other means to access the root account,
DEBUG: buildserver > and you are sure this is what you want,
DEBUG: buildserver > you may bypass this check by setting an environment variable
DEBUG: buildserver > (export SUDO_FORCE_REMOVE=yes).
DEBUG: buildserver >
DEBUG: buildserver > Refusing to remove sudo.
DEBUG: buildserver > dpkg: error processing package sudo (--purge):
DEBUG: buildserver > subprocess installed pre-removal script returned error exit status 1
DEBUG: buildserver > Errors were encountered while processing:
DEBUG: buildserver > sudo
DEBUG: buildserver > E: Sub-process /usr/bin/dpkg returned an error code (1)
2017-12-14 14:42:09 +01:00
p = FDroidPopen ( [ ' sudo ' , ' passwd ' , ' --lock ' , ' root ' ] )
if p . returncode != 0 :
raise BuildException ( " Error locking root account for %s : %s " %
( app . id , build . versionName ) , p . output )
2017-12-18 10:02:19 +01:00
p = FDroidPopen ( [ ' sudo ' , ' SUDO_FORCE_REMOVE=yes ' , ' dpkg ' , ' --purge ' , ' sudo ' ] )
2017-12-07 22:26:13 +01:00
if p . returncode != 0 :
raise BuildException ( " Error removing sudo for %s : %s " %
( app . id , build . versionName ) , p . output )
2017-03-21 23:51:15 +01:00
log_path = os . path . join ( log_dir ,
common . get_toolsversion_logname ( app , build ) )
with open ( log_path , ' w ' ) as f :
2018-01-17 17:13:12 +01:00
f . write ( common . get_android_tools_version_log ( build . ndk_path ( ) ) )
2017-06-28 23:01:45 +02:00
else :
if build . sudo :
2019-01-15 15:14:09 +01:00
logging . warning ( ' %s : %s runs this on the buildserver with sudo: \n \t %s \n These commands were skipped because fdroid build is not running on a dedicated build server. '
2017-06-28 23:01:45 +02:00
% ( app . id , build . versionName , build . sudo ) )
2017-03-21 23:51:15 +01:00
2012-02-21 01:01:07 +01:00
# Prepare the source code...
2015-11-28 17:55:27 +01:00
root_dir , srclibpaths = common . prepare_source ( vcs , app , build ,
2014-05-06 19:50:52 +02:00
build_dir , srclib_dir ,
2015-07-14 12:32:39 +02:00
extlib_dir , onserver , refresh )
2012-02-21 01:01:07 +01:00
2013-10-10 15:48:39 +02:00
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
2015-01-02 22:26:22 +01:00
gradletasks = [ ]
2016-02-15 13:01:38 +01:00
bmethod = build . build_method ( )
if bmethod == ' maven ' :
2014-01-27 16:45:30 +01:00
logging . info ( " Cleaning Maven project... " )
2013-10-31 16:37:39 +01:00
cmd = [ config [ ' mvn3 ' ] , ' clean ' , ' -Dandroid.sdk.path= ' + config [ ' sdk_path ' ] ]
2013-10-10 15:48:39 +02:00
2015-11-28 17:55:27 +01:00
if ' @ ' in build . maven :
maven_dir = os . path . join ( root_dir , build . maven . split ( ' @ ' , 1 ) [ 1 ] )
2013-12-07 13:15:51 +01:00
maven_dir = os . path . normpath ( maven_dir )
2013-10-23 19:52:17 +02:00
else :
maven_dir = root_dir
2013-11-01 12:10:57 +01:00
p = FDroidPopen ( cmd , cwd = maven_dir )
2014-01-10 20:39:39 +01:00
2016-02-15 13:01:38 +01:00
elif bmethod == ' gradle ' :
2014-01-16 00:20:26 +01:00
2014-01-27 16:45:30 +01:00
logging . info ( " Cleaning Gradle project... " )
2015-01-02 22:26:22 +01:00
2015-11-28 17:55:27 +01:00
if build . preassemble :
gradletasks + = build . preassemble
2015-01-02 22:26:22 +01:00
2015-11-28 17:55:27 +01:00
flavours = build . gradle
2015-01-02 22:26:22 +01:00
if flavours == [ ' yes ' ] :
flavours = [ ]
2017-08-26 12:55:34 +02:00
flavours_cmd = ' ' . join ( [ transform_first_char ( flav , str . upper ) for flav in flavours ] )
2015-01-02 22:26:22 +01:00
gradletasks + = [ ' assemble ' + flavours_cmd + ' Release ' ]
2013-10-10 15:48:39 +02:00
2015-01-02 22:26:22 +01:00
cmd = [ config [ ' gradle ' ] ]
2015-11-28 17:55:27 +01:00
if build . gradleprops :
2016-02-15 20:22:17 +01:00
cmd + = [ ' -P ' + kv for kv in build . gradleprops ]
2015-08-25 00:54:05 +02:00
2015-04-10 00:14:52 +02:00
cmd + = [ ' clean ' ]
2018-07-08 17:34:17 +02:00
p = FDroidPopen ( cmd , cwd = root_dir , envs = { " GRADLE_VERSION_DIR " : config [ ' gradle_version_dir ' ] , " CACHEDIR " : config [ ' cachedir ' ] } )
2014-01-10 20:39:39 +01:00
2017-06-20 14:16:31 +02:00
elif bmethod == ' buildozer ' :
pass
2016-02-15 13:01:38 +01:00
elif bmethod == ' ant ' :
2014-01-27 16:45:30 +01:00
logging . info ( " Cleaning Ant project... " )
2014-01-10 20:39:39 +01:00
p = FDroidPopen ( [ ' ant ' , ' clean ' ] , cwd = root_dir )
2013-10-16 23:17:51 +02:00
if p is not None and p . returncode != 0 :
raise BuildException ( " Error cleaning %s : %s " %
2016-11-23 17:52:04 +01:00
( app . id , build . versionName ) , p . output )
2013-10-10 17:52:17 +02:00
2014-03-05 12:32:36 +01:00
for root , dirs , files in os . walk ( build_dir ) :
2015-10-08 12:08:21 +02:00
def del_dirs ( dl ) :
for d in dl :
2020-05-21 07:50:08 +02:00
shutil . rmtree ( os . path . join ( root , d ) , ignore_errors = True )
2015-10-08 12:08:21 +02:00
def del_files ( fl ) :
for f in fl :
if f in files :
os . remove ( os . path . join ( root , f ) )
2019-12-23 02:02:28 +01:00
if any ( f in files for f in [ ' build.gradle ' , ' build.gradle.kts ' , ' settings.gradle ' , ' settings.gradle.kts ' ] ) :
2015-10-08 12:08:21 +02:00
# Even when running clean, gradle stores task/artifact caches in
# .gradle/ as binary files. To avoid overcomplicating the scanner,
# manually delete them, just like `gradle clean` should have removed
2018-01-10 19:06:32 +01:00
# the build/* dirs.
del_dirs ( [ os . path . join ( ' build ' , ' android-profile ' ) ,
os . path . join ( ' build ' , ' generated ' ) ,
os . path . join ( ' build ' , ' intermediates ' ) ,
os . path . join ( ' build ' , ' outputs ' ) ,
os . path . join ( ' build ' , ' reports ' ) ,
os . path . join ( ' build ' , ' tmp ' ) ,
2020-05-21 07:50:35 +02:00
os . path . join ( ' buildSrc ' , ' build ' ) ,
2018-01-10 19:06:32 +01:00
' .gradle ' ] )
2015-10-08 12:08:21 +02:00
del_files ( [ ' gradlew ' , ' gradlew.bat ' ] )
if ' pom.xml ' in files :
del_dirs ( [ ' target ' ] )
if any ( f in files for f in [ ' ant.properties ' , ' project.properties ' , ' build.xml ' ] ) :
del_dirs ( [ ' bin ' , ' gen ' ] )
if ' jni ' in dirs :
del_dirs ( [ ' obj ' ] )
2014-03-05 12:32:36 +01:00
2015-01-06 14:40:31 +01:00
if options . skipscan :
2015-11-28 17:55:27 +01:00
if build . scandelete :
2015-01-06 14:40:31 +01:00
raise BuildException ( " Refusing to skip source scan since scandelete is present " )
else :
2014-02-16 17:40:54 +01:00
# Scan before building...
logging . info ( " Scanning source for common problems... " )
2020-06-03 18:34:47 +02:00
scanner . options = options # pass verbose through
2017-05-25 20:11:14 +02:00
count = scanner . scan_source ( build_dir , build )
2014-04-03 16:04:06 +02:00
if count > 0 :
if force :
2017-09-13 17:33:57 +02:00
logging . warning ( ngettext ( ' Scanner found {} problem ' ,
' Scanner found {} problems ' , count ) . format ( count ) )
2014-04-03 16:04:06 +02:00
else :
2017-09-13 17:33:57 +02:00
raise BuildException ( ngettext (
" Can ' t build due to {} error while scanning " ,
" Can ' t build due to {} errors while scanning " , count ) . format ( count ) )
2012-02-21 01:01:07 +01:00
2014-01-28 22:13:18 +01:00
if not options . notarball :
# Build the source tarball right before we build the release...
2014-01-29 13:57:57 +01:00
logging . info ( " Creating source tarball... " )
2015-11-28 17:55:27 +01:00
tarname = common . getsrcname ( app , build )
2014-01-28 22:13:18 +01:00
tarball = tarfile . open ( os . path . join ( tmp_dir , tarname ) , " w:gz " )
2014-05-01 00:23:57 +02:00
2018-07-25 15:31:28 +02:00
def tarexc ( t ) :
return None if any ( t . name . endswith ( s ) for s in [ ' .svn ' , ' .git ' , ' .hg ' , ' .bzr ' ] ) else t
tarball . add ( build_dir , tarname , filter = tarexc )
2014-01-28 22:13:18 +01:00
tarball . close ( )
2012-02-21 01:01:07 +01:00
2013-03-20 10:30:56 +01:00
# Run a build command if one is required...
2015-11-28 17:55:27 +01:00
if build . build :
2014-05-31 23:10:16 +02:00
logging . info ( " Running ' build ' commands in %s " % root_dir )
2015-11-28 17:55:27 +01:00
cmd = common . replace_config_vars ( build . build , build )
2014-05-31 23:10:16 +02:00
2013-06-15 19:21:28 +02:00
# Substitute source library paths into commands...
2013-11-16 22:49:37 +01:00
for name , number , libpath in srclibpaths :
2020-06-24 20:50:21 +02:00
cmd = cmd . replace ( ' $$ ' + name + ' $$ ' , os . path . join ( os . getcwd ( ) , libpath ) )
2013-10-09 23:36:24 +02:00
2013-11-08 20:44:27 +01:00
p = FDroidPopen ( [ ' bash ' , ' -x ' , ' -c ' , cmd ] , cwd = root_dir )
2013-12-30 17:04:16 +01:00
2013-03-20 10:30:56 +01:00
if p . returncode != 0 :
2013-10-09 23:36:24 +02:00
raise BuildException ( " Error running build command for %s : %s " %
2016-11-23 17:52:04 +01:00
( app . id , build . versionName ) , p . output )
2013-03-20 10:30:56 +01:00
2012-02-21 01:01:07 +01:00
# Build native stuff if required...
2015-11-28 17:55:27 +01:00
if build . buildjni and build . buildjni != [ ' no ' ] :
2014-05-31 23:10:16 +02:00
logging . info ( " Building the native code " )
2015-11-28 17:55:27 +01:00
jni_components = build . buildjni
2014-05-31 23:10:16 +02:00
2014-02-15 10:56:35 +01:00
if jni_components == [ ' yes ' ] :
2012-03-07 07:46:56 +01:00
jni_components = [ ' ' ]
2015-11-28 17:55:27 +01:00
cmd = [ os . path . join ( ndk_path , " ndk-build " ) , " -j1 " ]
2012-03-07 07:46:56 +01:00
for d in jni_components :
2014-04-03 16:10:54 +02:00
if d :
logging . info ( " Building native code in ' %s ' " % d )
else :
logging . info ( " Building native code in the main project " )
2015-09-15 03:12:15 +02:00
manifest = os . path . join ( root_dir , d , ' AndroidManifest.xml ' )
2013-03-31 19:33:18 +02:00
if os . path . exists ( manifest ) :
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open ( manifest , ' U ' ) . read ( )
open ( manifest , ' w ' ) . write ( manifest_text )
# In case the AM.xml read was big, free the memory
del manifest_text
2014-05-01 00:23:57 +02:00
p = FDroidPopen ( cmd , cwd = os . path . join ( root_dir , d ) )
2013-10-16 23:17:51 +02:00
if p . returncode != 0 :
2016-11-23 17:52:04 +01:00
raise BuildException ( " NDK build failed for %s : %s " % ( app . id , build . versionName ) , p . output )
2012-02-21 01:01:07 +01:00
2013-09-04 16:45:19 +02:00
p = None
2012-02-21 01:01:07 +01:00
# Build the release...
2016-02-15 13:01:38 +01:00
if bmethod == ' maven ' :
2014-01-27 17:04:22 +01:00
logging . info ( " Building Maven project... " )
2013-10-23 19:52:17 +02:00
2015-11-28 17:55:27 +01:00
if ' @ ' in build . maven :
maven_dir = os . path . join ( root_dir , build . maven . split ( ' @ ' , 1 ) [ 1 ] )
2013-10-23 19:52:17 +02:00
else :
maven_dir = root_dir
2013-12-11 17:29:38 +01:00
mvncmd = [ config [ ' mvn3 ' ] , ' -Dandroid.sdk.path= ' + config [ ' sdk_path ' ] ,
2014-05-06 19:50:52 +02:00
' -Dmaven.jar.sign.skip=true ' , ' -Dmaven.test.skip=true ' ,
' -Dandroid.sign.debug=false ' , ' -Dandroid.release=true ' ,
' package ' ]
2015-11-28 17:55:27 +01:00
if build . target :
target = build . target . split ( ' - ' ) [ 1 ]
2015-07-30 22:13:12 +02:00
common . regsub_file ( r ' <platform>[0-9]*</platform> ' ,
r ' <platform> %s </platform> ' % target ,
os . path . join ( root_dir , ' pom.xml ' ) )
2015-11-28 17:55:27 +01:00
if ' @ ' in build . maven :
2015-07-30 22:13:12 +02:00
common . regsub_file ( r ' <platform>[0-9]*</platform> ' ,
r ' <platform> %s </platform> ' % target ,
os . path . join ( maven_dir , ' pom.xml ' ) )
2013-06-10 22:50:15 +02:00
2013-11-04 20:19:31 +01:00
p = FDroidPopen ( mvncmd , cwd = maven_dir )
2013-09-02 19:05:13 +02:00
2013-11-09 12:21:43 +01:00
bindir = os . path . join ( root_dir , ' target ' )
2017-06-20 14:16:31 +02:00
elif bmethod == ' buildozer ' :
logging . info ( " Building Kivy project using buildozer... " )
# parse buildozer.spez
spec = os . path . join ( root_dir , ' buildozer.spec ' )
if not os . path . exists ( spec ) :
raise BuildException ( " Expected to find buildozer-compatible spec at {0} "
. format ( spec ) )
defaults = { ' orientation ' : ' landscape ' , ' icon ' : ' ' ,
2017-06-20 15:56:15 +02:00
' permissions ' : ' ' , ' android.api ' : " 19 " }
2017-06-20 14:16:31 +02:00
bconfig = ConfigParser ( defaults , allow_no_value = True )
bconfig . read ( spec )
# update spec with sdk and ndk locations to prevent buildozer from
# downloading.
loc_ndk = common . env [ ' ANDROID_NDK ' ]
loc_sdk = common . env [ ' ANDROID_SDK ' ]
if loc_ndk == ' $ANDROID_NDK ' :
2017-06-20 15:56:15 +02:00
loc_ndk = loc_sdk + ' /ndk-bundle '
2017-06-20 14:16:31 +02:00
bc_ndk = None
bc_sdk = None
try :
2017-06-20 15:56:15 +02:00
bc_ndk = bconfig . get ( ' app ' , ' android.sdk_path ' )
except Exception :
2017-06-20 14:16:31 +02:00
pass
try :
2017-06-20 15:56:15 +02:00
bc_sdk = bconfig . get ( ' app ' , ' android.ndk_path ' )
except Exception :
2017-06-20 14:16:31 +02:00
pass
if bc_sdk is None :
2017-06-20 15:56:15 +02:00
bconfig . set ( ' app ' , ' android.sdk_path ' , loc_sdk )
2017-06-20 14:16:31 +02:00
if bc_ndk is None :
2017-06-20 15:56:15 +02:00
bconfig . set ( ' app ' , ' android.ndk_path ' , loc_ndk )
2017-06-20 14:16:31 +02:00
2017-06-20 15:56:15 +02:00
fspec = open ( spec , ' w ' )
2017-06-20 14:16:31 +02:00
bconfig . write ( fspec )
fspec . close ( )
logging . info ( " sdk_path = %s " % loc_sdk )
logging . info ( " ndk_path = %s " % loc_ndk )
p = None
# execute buildozer
2017-06-20 15:56:15 +02:00
cmd = [ ' buildozer ' , ' android ' , ' release ' ]
2017-06-20 14:16:31 +02:00
try :
p = FDroidPopen ( cmd , cwd = root_dir )
2017-06-20 15:56:15 +02:00
except Exception :
2017-06-20 14:16:31 +02:00
pass
# buidozer not installed ? clone repo and run
2017-06-20 15:56:15 +02:00
if ( p is None or p . returncode != 0 ) :
cmd = [ ' git ' , ' clone ' , ' https://github.com/kivy/buildozer.git ' ]
2017-06-20 14:16:31 +02:00
p = subprocess . Popen ( cmd , cwd = root_dir , shell = False )
p . wait ( )
if p . returncode != 0 :
raise BuildException ( " Distribute build failed " )
2017-06-20 15:56:15 +02:00
cmd = [ ' python ' , ' buildozer/buildozer/scripts/client.py ' , ' android ' , ' release ' ]
2017-06-20 14:16:31 +02:00
p = FDroidPopen ( cmd , cwd = root_dir )
# expected to fail.
# Signing will fail if not set by environnment vars (cf. p4a docs).
# But the unsigned apk will be ok.
p . returncode = 0
2016-02-15 13:01:38 +01:00
elif bmethod == ' gradle ' :
2014-01-27 17:04:22 +01:00
logging . info ( " Building Gradle project... " )
2013-08-22 18:28:30 +02:00
2015-08-25 00:54:05 +02:00
cmd = [ config [ ' gradle ' ] ]
2015-11-28 17:55:27 +01:00
if build . gradleprops :
2016-02-15 20:22:17 +01:00
cmd + = [ ' -P ' + kv for kv in build . gradleprops ]
2015-08-25 00:54:05 +02:00
cmd + = gradletasks
2015-01-02 22:26:22 +01:00
2018-07-08 17:34:17 +02:00
p = FDroidPopen ( cmd , cwd = root_dir , envs = { " GRADLE_VERSION_DIR " : config [ ' gradle_version_dir ' ] , " CACHEDIR " : config [ ' cachedir ' ] } )
2013-09-02 19:05:13 +02:00
2016-02-15 13:01:38 +01:00
elif bmethod == ' ant ' :
2014-01-27 17:04:22 +01:00
logging . info ( " Building Ant project... " )
2013-10-16 23:17:51 +02:00
cmd = [ ' ant ' ]
2015-11-28 17:55:27 +01:00
if build . antcommands :
cmd + = build . antcommands
2012-02-21 01:01:07 +01:00
else :
2013-10-16 23:17:51 +02:00
cmd + = [ ' release ' ]
2013-11-04 20:19:31 +01:00
p = FDroidPopen ( cmd , cwd = root_dir )
2013-10-16 23:34:51 +02:00
2013-11-09 12:21:43 +01:00
bindir = os . path . join ( root_dir , ' bin ' )
2014-01-28 22:14:53 +01:00
if p is not None and p . returncode != 0 :
2016-11-23 17:52:04 +01:00
raise BuildException ( " Build failed for %s : %s " % ( app . id , build . versionName ) , p . output )
logging . info ( " Successfully built version " + build . versionName + ' of ' + app . id )
2012-02-21 01:01:07 +01:00
2016-02-15 13:01:38 +01:00
omethod = build . output_method ( )
if omethod == ' maven ' :
2013-11-04 20:19:31 +01:00
stdout_apk = ' \n ' . join ( [
2014-08-12 12:56:15 +02:00
line for line in p . output . splitlines ( ) if any (
a in line for a in ( ' .apk ' , ' .ap_ ' , ' .jar ' ) ) ] )
2012-06-28 13:52:35 +02:00
m = re . match ( r " .*^ \ [INFO \ ] .*apkbuilder.*/([^/]*) \ .apk " ,
2014-05-06 19:50:52 +02:00
stdout_apk , re . S | re . M )
2012-06-28 13:52:35 +02:00
if not m :
2013-10-23 19:53:07 +02:00
m = re . match ( r " .*^ \ [INFO \ ] Creating additional unsigned apk file .*/([^/]+) \ .apk[^l] " ,
2014-05-06 19:50:52 +02:00
stdout_apk , re . S | re . M )
2012-08-13 11:16:25 +02:00
if not m :
2013-11-08 18:56:28 +01:00
m = re . match ( r ' .*^ \ [INFO \ ] [^$]*aapt \ [package,[^$]* ' + bindir + r ' /([^/]+) \ .ap[_k][, \ ]] ' ,
2014-05-06 19:50:52 +02:00
stdout_apk , re . S | re . M )
2014-08-12 12:56:15 +02:00
if not m :
m = re . match ( r " .*^ \ [INFO \ ] Building jar: .*/ " + bindir + r " /(.+) \ .jar " ,
stdout_apk , re . S | re . M )
2012-06-28 13:52:35 +02:00
if not m :
2012-06-25 10:22:11 +02:00
raise BuildException ( ' Failed to find output ' )
2012-06-28 13:52:35 +02:00
src = m . group ( 1 )
2012-02-21 01:01:07 +01:00
src = os . path . join ( bindir , src ) + ' .apk '
2017-06-20 14:16:31 +02:00
elif omethod == ' buildozer ' :
src = None
for apks_dir in [
os . path . join ( root_dir , ' .buildozer ' , ' android ' , ' platform ' , ' build ' , ' dists ' , bconfig . get ( ' app ' , ' title ' ) , ' bin ' ) ,
] :
for apkglob in [ ' *-release-unsigned.apk ' , ' *-unsigned.apk ' , ' *.apk ' ] :
apks = glob . glob ( os . path . join ( apks_dir , apkglob ) )
if len ( apks ) > 1 :
raise BuildException ( ' More than one resulting apks found in %s ' % apks_dir ,
' \n ' . join ( apks ) )
if len ( apks ) == 1 :
src = apks [ 0 ]
break
if src is not None :
break
if src is None :
raise BuildException ( ' Failed to find any output apks ' )
2016-02-15 13:01:38 +01:00
elif omethod == ' gradle ' :
2015-12-24 16:41:39 +01:00
src = None
2017-08-26 12:55:34 +02:00
apk_dirs = [
2017-08-26 17:03:13 +02:00
# gradle plugin >= 3.0
2017-08-26 12:55:34 +02:00
os . path . join ( root_dir , ' build ' , ' outputs ' , ' apk ' , ' release ' ) ,
2017-08-26 17:03:13 +02:00
# gradle plugin < 3.0 and >= 0.11
2017-08-26 12:55:34 +02:00
os . path . join ( root_dir , ' build ' , ' outputs ' , ' apk ' ) ,
2017-08-26 17:03:13 +02:00
# really old path
2017-08-26 12:55:34 +02:00
os . path . join ( root_dir , ' build ' , ' apk ' ) ,
]
2017-08-26 17:03:13 +02:00
# If we build with gradle flavours with gradle plugin >= 3.0 the apk will be in
# a subdirectory corresponding to the flavour command used, but with different
# capitalization.
2017-08-26 12:55:34 +02:00
if flavours_cmd :
apk_dirs . append ( os . path . join ( root_dir , ' build ' , ' outputs ' , ' apk ' , transform_first_char ( flavours_cmd , str . lower ) , ' release ' ) )
for apks_dir in apk_dirs :
2015-12-31 09:37:08 +01:00
for apkglob in [ ' *-release-unsigned.apk ' , ' *-unsigned.apk ' , ' *.apk ' ] :
apks = glob . glob ( os . path . join ( apks_dir , apkglob ) )
if len ( apks ) > 1 :
raise BuildException ( ' More than one resulting apks found in %s ' % apks_dir ,
' \n ' . join ( apks ) )
if len ( apks ) == 1 :
src = apks [ 0 ]
break
if src is not None :
2015-12-24 16:41:39 +01:00
break
if src is None :
raise BuildException ( ' Failed to find any output apks ' )
2014-07-09 11:08:11 +02:00
2016-02-15 13:01:38 +01:00
elif omethod == ' ant ' :
2013-11-04 20:19:31 +01:00
stdout_apk = ' \n ' . join ( [
2014-07-01 18:04:41 +02:00
line for line in p . output . splitlines ( ) if ' .apk ' in line ] )
2013-11-04 20:19:31 +01:00
src = re . match ( r " .*^.*Creating (.+) for release.*$.* " , stdout_apk ,
2014-05-06 19:50:52 +02:00
re . S | re . M ) . group ( 1 )
2012-02-21 01:01:07 +01:00
src = os . path . join ( bindir , src )
2016-02-15 13:01:38 +01:00
elif omethod == ' raw ' :
2017-03-16 09:23:28 +01:00
output_path = common . replace_build_vars ( build . output , build )
globpath = os . path . join ( root_dir , output_path )
2016-02-15 13:01:38 +01:00
apks = glob . glob ( globpath )
if len ( apks ) > 1 :
raise BuildException ( ' Multiple apks match %s ' % globpath , ' \n ' . join ( apks ) )
if len ( apks ) < 1 :
raise BuildException ( ' No apks match %s ' % globpath )
src = os . path . normpath ( apks [ 0 ] )
2012-02-21 01:01:07 +01:00
2013-04-15 14:04:28 +02:00
# Make sure it's not debuggable...
2018-02-22 15:08:53 +01:00
if common . is_apk_and_debuggable ( src ) :
2013-04-15 14:04:28 +02:00
raise BuildException ( " APK is debuggable " )
2012-02-21 01:01:07 +01:00
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
2014-07-05 15:29:12 +02:00
logging . debug ( " Checking " + src )
2012-06-28 13:52:35 +02:00
if not os . path . exists ( src ) :
raise BuildException ( " Unsigned apk is not at expected location of " + src )
2013-08-08 13:00:02 +02:00
2016-10-31 19:53:55 +01:00
if common . get_file_extension ( src ) == ' apk ' :
vercode , version = get_metadata_from_apk ( app , build , src )
2018-05-21 22:42:02 +02:00
if version != build . versionName or vercode != build . versionCode :
2016-10-31 19:53:55 +01:00
raise BuildException ( ( " Unexpected version/version code in output; "
" APK: ' %s ' / ' %s ' , "
" Expected: ' %s ' / ' %s ' " )
2016-11-23 17:52:04 +01:00
% ( version , str ( vercode ) , build . versionName ,
str ( build . versionCode ) ) )
2020-06-15 20:03:19 +02:00
if ( options . scan_binary or config . get ( ' scan_binary ' ) ) and not options . skipscan :
if scanner . scan_binary ( src ) :
raise BuildException ( " Found blacklisted packages in final apk! " )
2014-10-24 23:20:42 +02:00
2012-02-21 01:01:07 +01:00
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
2016-10-31 16:51:34 +01:00
dest = os . path . join ( output_dir , common . get_release_filename ( app , build ) )
2012-02-21 01:01:07 +01:00
shutil . copyfile ( src , dest )
# Move the source tarball into the output directory...
2014-01-28 22:13:18 +01:00
if output_dir != tmp_dir and not options . notarball :
2013-10-19 12:18:48 +02:00
shutil . move ( os . path . join ( tmp_dir , tarname ) ,
2014-05-01 00:23:57 +02:00
os . path . join ( output_dir , tarname ) )
2012-02-21 01:01:07 +01:00
2017-03-21 23:51:15 +01:00
def trybuild ( app , build , build_dir , output_dir , log_dir , also_check_dir ,
srclib_dir , extlib_dir , tmp_dir , repo_dir , vcs , test ,
server , force , onserver , refresh ) :
2012-02-26 15:09:25 +01:00
"""
Build a particular version of an application , if it needs building .
2013-10-24 10:42:51 +02:00
: param output_dir : The directory where the build output will go . Usually
this is the ' unsigned ' directory .
: param repo_dir : The repo directory - used for checking if the build is
necessary .
2017-03-26 01:41:39 +01:00
: param also_check_dir : An additional location for checking if the build
2013-10-24 10:42:51 +02:00
is necessary ( usually the archive repo )
: param test : True if building in test mode , in which case the build will
always happen , even if the output already exists . In test mode , the
output directory should be a temporary location , not any of the real
ones .
: returns : True if the build was done , False if it wasn ' t necessary.
2012-02-26 15:09:25 +01:00
"""
2016-10-31 16:51:34 +01:00
dest_file = common . get_release_filename ( app , build )
2013-10-19 12:18:48 +02:00
2016-10-31 16:51:34 +01:00
dest = os . path . join ( output_dir , dest_file )
dest_repo = os . path . join ( repo_dir , dest_file )
2012-02-26 15:09:25 +01:00
2013-10-24 10:42:51 +02:00
if not test :
if os . path . exists ( dest ) or os . path . exists ( dest_repo ) :
2013-05-09 21:09:17 +02:00
return False
2013-10-24 10:42:51 +02:00
if also_check_dir :
2016-10-31 16:51:34 +01:00
dest_also = os . path . join ( also_check_dir , dest_file )
2013-10-24 10:42:51 +02:00
if os . path . exists ( dest_also ) :
return False
2015-11-28 17:55:27 +01:00
if build . disable and not options . force :
2012-02-26 15:09:25 +01:00
return False
2014-06-30 16:34:26 +02:00
logging . info ( " Building version %s ( %s ) of %s " % (
2016-11-23 17:52:04 +01:00
build . versionName , build . versionCode , app . id ) )
2012-02-26 15:09:25 +01:00
2012-02-26 18:14:15 +01:00
if server :
2012-08-31 15:48:50 +02:00
# When using server mode, still keep a local cache of the repo, by
# grabbing the source now.
2017-12-26 00:20:17 +01:00
vcs . gotorevision ( build . commit , refresh )
2012-08-31 15:48:50 +02:00
2017-03-21 23:51:15 +01:00
build_server ( app , build , vcs , build_dir , output_dir , log_dir , force )
2012-02-26 15:09:25 +01:00
else :
2017-03-21 23:51:15 +01:00
build_local ( app , build , vcs , build_dir , output_dir , log_dir , srclib_dir , extlib_dir , tmp_dir , force , onserver , refresh )
2012-02-26 15:09:25 +01:00
return True
2018-01-28 07:56:19 +01:00
def force_halt_build ( timeout ) :
2018-01-15 01:03:47 +01:00
""" Halt the currently running Vagrant VM, to be called from a Timer """
2018-01-28 07:56:19 +01:00
logging . error ( _ ( ' Force halting build after {0} sec timeout! ' ) . format ( timeout ) )
2018-01-20 22:16:39 +01:00
timeout_event . set ( )
2018-01-15 01:03:47 +01:00
vm = vmtools . get_build_vm ( ' builder ' )
vm . halt ( )
2012-02-26 15:09:25 +01:00
def parse_commandline ( ) :
2015-09-04 11:37:05 +02:00
""" Parse the command line. Returns options, parser. """
2019-01-29 11:23:45 +01:00
parser = argparse . ArgumentParser ( usage = " %(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]] " )
2015-09-12 08:42:50 +02:00
common . setup_global_opts ( parser )
2017-09-15 11:41:39 +02:00
parser . add_argument ( " appid " , nargs = ' * ' , help = _ ( " applicationId with optional versionCode in the form APPID[:VERCODE] " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -l " , " --latest " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Build only the latest version of each package " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -s " , " --stop " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Make the build stop on exceptions " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -t " , " --test " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Test mode - put output in the tmp directory only, and always build, even if the output already exists. " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --server " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Use build server " ) )
2018-01-30 20:45:03 +01:00
parser . add_argument ( " --reset-server " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Reset and create a brand new build server, even if the existing one appears to be ok. " ) )
2019-01-29 11:23:45 +01:00
# this option is internal API for telling fdroid that
# it's running inside a buildserver vm.
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --on-server " , dest = " onserver " , action = " store_true " , default = False ,
2019-01-29 11:23:45 +01:00
help = argparse . SUPPRESS )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --skip-scan " , dest = " skipscan " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Skip scanning the source code for binaries and other problems " ) )
2020-06-15 20:03:19 +02:00
parser . add_argument ( " --scan-binary " , action = " store_true " , default = False ,
help = _ ( " Scan the resulting APK(s) for known non-free classes. " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --no-tarball " , dest = " notarball " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Don ' t create a source tarball, useful when testing a build " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " --no-refresh " , dest = " refresh " , action = " store_false " , default = True ,
2017-09-13 18:03:57 +02:00
help = _ ( " Don ' t refresh the repository, useful when testing a build with no internet connection " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -f " , " --force " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode. " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -a " , " --all " , action = " store_true " , default = False ,
2017-09-13 18:03:57 +02:00
help = _ ( " Build all applications available " ) )
2015-09-04 11:37:05 +02:00
parser . add_argument ( " -w " , " --wiki " , default = False , action = " store_true " ,
2017-09-13 18:03:57 +02:00
help = _ ( " Update the wiki " ) )
2016-09-12 12:55:48 +02:00
metadata . add_metadata_arguments ( parser )
2015-09-04 11:37:05 +02:00
options = parser . parse_args ( )
2016-09-12 12:55:48 +02:00
metadata . warnings_action = options . W
2012-02-26 15:09:25 +01:00
2014-03-05 00:36:32 +01:00
# Force --stop with --on-server to get correct exit code
2013-10-21 22:16:41 +02:00
if options . onserver :
options . stop = True
2012-03-09 16:17:16 +01:00
if options . force and not options . test :
2015-09-04 11:37:05 +02:00
parser . error ( " option %s : Force is only allowed in test mode " % " force " )
2012-03-09 16:17:16 +01:00
2015-09-04 11:37:05 +02:00
return options , parser
2012-02-26 15:09:25 +01:00
2016-11-15 21:55:06 +01:00
2012-03-07 07:46:56 +01:00
options = None
2013-11-01 12:10:57 +01:00
config = None
2017-02-09 23:49:42 +01:00
buildserverid = None
2018-01-22 14:00:16 +01:00
fdroidserverid = None
start_timestamp = time . gmtime ( )
2020-02-18 23:16:18 +01:00
status_output = None
2018-01-20 22:16:39 +01:00
timeout_event = threading . Event ( )
2012-02-26 15:09:25 +01:00
2014-05-01 00:23:57 +02:00
2012-02-26 15:09:25 +01:00
def main ( ) :
2018-01-22 14:00:16 +01:00
global options , config , buildserverid , fdroidserverid
2013-05-09 21:09:17 +02:00
2015-09-04 11:37:05 +02:00
options , parser = parse_commandline ( )
2015-08-04 16:09:44 +02:00
2015-08-05 15:18:22 +02:00
# The defaults for .fdroid.* metadata that is included in a git repo are
# different than for the standard metadata/ layout because expectations
# are different. In this case, the most common user will be the app
# developer working on the latest update of the app on their own machine.
2016-03-27 21:36:16 +02:00
local_metadata_files = common . get_local_metadata_files ( )
2015-08-05 15:18:22 +02:00
if len ( local_metadata_files ) == 1 : # there is local metadata in an app's source
config = dict ( common . default_config )
# `fdroid build` should build only the latest version by default since
# most of the time the user will be building the most recent update
if not options . all :
options . latest = True
elif len ( local_metadata_files ) > 1 :
2015-08-04 16:09:44 +02:00
raise FDroidException ( " Only one local metadata file allowed! Found: "
2015-08-05 15:18:22 +02:00
+ " " . join ( local_metadata_files ) )
else :
if not os . path . isdir ( ' metadata ' ) and len ( local_metadata_files ) == 0 :
raise FDroidException ( " No app metadata found, nothing to process! " )
if not options . appid and not options . all :
parser . error ( " option %s : If you really want to build all the apps, use --all " % " all " )
2013-12-22 21:31:35 +01:00
2013-11-01 12:10:57 +01:00
config = common . read_config ( options )
2013-10-31 16:37:39 +01:00
if config [ ' build_server_always ' ] :
2012-09-03 12:48:47 +02:00
options . server = True
2018-01-30 20:45:03 +01:00
if options . reset_server and not options . server :
parser . error ( " option %s : Using --reset-server without --server makes no sense " % " reset-server " )
2012-02-26 15:09:25 +01:00
2018-06-22 23:26:15 +02:00
if options . onserver or not options . server :
for d in [ ' build-tools ' , ' platform-tools ' , ' tools ' ] :
if not os . path . isdir ( os . path . join ( config [ ' sdk_path ' ] , d ) ) :
raise FDroidException ( _ ( " Android SDK ' {path} ' does not have ' {dirname} ' installed! " )
. format ( path = config [ ' sdk_path ' ] , dirname = d ) )
2012-02-26 15:09:25 +01:00
log_dir = ' logs '
if not os . path . isdir ( log_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating log directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( log_dir )
tmp_dir = ' tmp '
if not os . path . isdir ( tmp_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating temporary directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( tmp_dir )
if options . test :
output_dir = tmp_dir
2012-01-22 00:10:59 +01:00
else :
2012-02-26 15:09:25 +01:00
output_dir = ' unsigned '
if not os . path . isdir ( output_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating output directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( output_dir )
2018-07-12 23:52:46 +02:00
binaries_dir = os . path . join ( output_dir , ' binaries ' )
2012-02-26 15:09:25 +01:00
2013-10-31 16:37:39 +01:00
if config [ ' archive_older ' ] != 0 :
2013-05-09 21:09:17 +02:00
also_check_dir = ' archive '
else :
also_check_dir = None
2020-02-20 09:02:39 +01:00
if options . onserver :
status_output = dict ( ) # HACK dummy placeholder
else :
status_output = common . setup_status_output ( start_timestamp )
2020-02-18 23:16:18 +01:00
2012-02-26 15:09:25 +01:00
repo_dir = ' repo '
build_dir = ' build '
if not os . path . isdir ( build_dir ) :
2014-01-27 17:04:22 +01:00
logging . info ( " Creating build directory " )
2012-02-26 15:09:25 +01:00
os . makedirs ( build_dir )
2013-05-20 13:34:03 +02:00
srclib_dir = os . path . join ( build_dir , ' srclib ' )
2012-02-26 15:09:25 +01:00
extlib_dir = os . path . join ( build_dir , ' extlib ' )
2014-05-20 23:44:47 +02:00
# Read all app and srclib metadata
2016-11-07 21:47:53 +01:00
pkgs = common . read_pkg_args ( options . appid , True )
2017-12-26 00:20:17 +01:00
allapps = metadata . read_metadata ( not options . onserver , pkgs , options . refresh , sort_by_time = True )
2015-09-04 11:37:05 +02:00
apps = common . read_app_args ( options . appid , allapps , True )
2016-11-07 21:47:53 +01:00
2016-03-14 23:23:19 +01:00
for appid , app in list ( apps . items ( ) ) :
2015-11-28 13:09:47 +01:00
if ( app . Disabled and not options . force ) or not app . RepoType or not app . builds :
2014-08-16 12:46:02 +02:00
del apps [ appid ]
2013-12-11 19:08:15 +01:00
2014-08-16 12:46:02 +02:00
if not apps :
2014-07-07 15:41:32 +02:00
raise FDroidException ( " No apps to process. " )
2013-12-11 19:08:15 +01:00
2017-11-28 10:39:35 +01:00
# make sure enough open files are allowed to process everything
soft , hard = resource . getrlimit ( resource . RLIMIT_NOFILE )
if len ( apps ) > soft :
try :
soft = len ( apps ) * 2
if soft > hard :
soft = hard
resource . setrlimit ( resource . RLIMIT_NOFILE , ( soft , hard ) )
logging . debug ( _ ( ' Set open file limit to {integer} ' )
. format ( integer = soft ) )
except ( OSError , ValueError ) as e :
logging . warning ( _ ( ' Setting open file limit failed: ' ) + str ( e ) )
2013-12-11 19:08:15 +01:00
if options . latest :
2016-01-04 17:43:22 +01:00
for app in apps . values ( ) :
2015-11-28 13:09:47 +01:00
for build in reversed ( app . builds ) :
2015-11-28 17:55:27 +01:00
if build . disable and not options . force :
2014-01-20 09:58:17 +01:00
continue
2015-11-28 13:09:47 +01:00
app . builds = [ build ]
2014-01-20 09:58:17 +01:00
break
2012-02-26 15:09:25 +01:00
2013-05-31 08:49:39 +02:00
if options . wiki :
import mwclient
2013-10-31 16:37:39 +01:00
site = mwclient . Site ( ( config [ ' wiki_protocol ' ] , config [ ' wiki_server ' ] ) ,
2014-05-06 19:50:52 +02:00
path = config [ ' wiki_path ' ] )
2013-10-31 16:37:39 +01:00
site . login ( config [ ' wiki_user ' ] , config [ ' wiki_password ' ] )
2013-05-31 08:49:39 +02:00
2012-02-26 15:09:25 +01:00
# Build applications...
2020-02-19 20:18:18 +01:00
failed_builds = [ ]
2012-02-26 15:09:25 +01:00
build_succeeded = [ ]
2020-02-19 20:18:18 +01:00
status_output [ ' failedBuilds ' ] = failed_builds
2020-02-18 23:16:18 +01:00
status_output [ ' successfulBuilds ' ] = build_succeeded
2018-02-12 11:45:49 +01:00
# Only build for 36 hours, then stop gracefully.
endtime = time . time ( ) + 36 * 60 * 60
2018-01-19 22:35:06 +01:00
max_build_time_reached = False
2016-01-04 17:02:28 +01:00
for appid , app in apps . items ( ) :
2010-11-11 01:03:39 +01:00
2013-05-31 08:49:39 +02:00
first = True
2013-04-23 14:29:00 +02:00
2015-11-28 17:55:27 +01:00
for build in app . builds :
2018-01-19 22:35:06 +01:00
if time . time ( ) > endtime :
max_build_time_reached = True
break
2018-01-28 07:56:19 +01:00
# Enable watchdog timer (2 hours by default).
if build . timeout is None :
timeout = 7200
else :
timeout = int ( build . timeout )
if options . server and timeout > 0 :
logging . debug ( _ ( ' Setting {0} sec timeout for this build ' ) . format ( timeout ) )
timer = threading . Timer ( timeout , force_halt_build , [ timeout ] )
2018-02-07 20:46:29 +01:00
timeout_event . clear ( )
2018-01-15 01:03:47 +01:00
timer . start ( )
else :
timer = None
2013-05-04 11:10:12 +02:00
wikilog = None
2018-01-17 14:39:54 +01:00
build_starttime = common . get_wiki_timestamp ( )
2017-03-21 23:51:15 +01:00
tools_version_log = ' '
if not options . onserver :
2018-01-17 17:13:12 +01:00
tools_version_log = common . get_android_tools_version_log ( build . ndk_path ( ) )
2020-02-20 09:02:39 +01:00
common . write_running_status_json ( status_output )
2012-01-02 15:09:20 +01:00
try :
2013-05-31 08:49:39 +02:00
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first :
2016-11-07 21:47:53 +01:00
vcs , build_dir = common . setup_vcs ( app )
2013-05-31 08:49:39 +02:00
first = False
2017-12-02 22:41:08 +01:00
logging . info ( " Using %s " % vcs . clientversion ( ) )
2016-11-23 17:52:04 +01:00
logging . debug ( " Checking " + build . versionName )
2017-03-21 23:51:15 +01:00
if trybuild ( app , build , build_dir , output_dir , log_dir ,
2014-05-06 19:50:52 +02:00
also_check_dir , srclib_dir , extlib_dir ,
tmp_dir , repo_dir , vcs , options . test ,
options . server , options . force ,
2015-07-14 12:32:39 +02:00
options . onserver , options . refresh ) :
2017-03-21 23:51:15 +01:00
toolslog = os . path . join ( log_dir ,
common . get_toolsversion_logname ( app , build ) )
if not options . onserver and os . path . exists ( toolslog ) :
with open ( toolslog , ' r ' ) as f :
tools_version_log = ' ' . join ( f . readlines ( ) )
os . remove ( toolslog )
2015-01-31 16:36:26 +01:00
2015-11-28 13:09:47 +01:00
if app . Binaries is not None :
2015-01-31 16:36:26 +01:00
# This is an app where we build from source, and
# verify the apk contents against a developer's
# binary. We get that binary now, and save it
# alongside our built one in the 'unsigend'
# directory.
2018-07-12 23:52:46 +02:00
if not os . path . isdir ( binaries_dir ) :
os . makedirs ( binaries_dir )
logging . info ( " Created directory for storing "
2018-07-12 23:43:19 +02:00
" developer supplied reference "
2018-07-12 23:52:46 +02:00
" binaries: ' {path} ' "
. format ( path = binaries_dir ) )
2015-11-28 13:09:47 +01:00
url = app . Binaries
2016-11-23 17:52:04 +01:00
url = url . replace ( ' % v ' , build . versionName )
url = url . replace ( ' %c ' , str ( build . versionCode ) )
2015-01-31 16:36:26 +01:00
logging . info ( " ...retrieving " + url )
2017-07-15 22:15:30 +02:00
of = re . sub ( r ' .apk$ ' , ' .binary.apk ' , common . get_release_filename ( app , build ) )
2018-07-12 23:52:46 +02:00
of = os . path . join ( binaries_dir , of )
2017-04-20 17:48:38 +02:00
try :
net . download_file ( url , local_filename = of )
except requests . exceptions . HTTPError as e :
2017-05-22 21:33:52 +02:00
raise FDroidException (
2018-06-27 01:05:45 +02:00
' Downloading Binaries from %s failed. ' % url ) from e
2015-01-31 16:36:26 +01:00
2018-06-12 16:18:21 +02:00
# Now we check whether the build can be verified to
2017-04-04 18:58:16 +02:00
# match the supplied binary or not. Should the
# comparison fail, we mark this build as a failure
# and remove everything from the unsigend folder.
with tempfile . TemporaryDirectory ( ) as tmpdir :
unsigned_apk = \
2017-04-22 12:04:32 +02:00
common . get_release_filename ( app , build )
unsigned_apk = \
os . path . join ( output_dir , unsigned_apk )
2017-04-04 18:58:16 +02:00
compare_result = \
2017-04-20 12:44:22 +02:00
common . verify_apks ( of , unsigned_apk , tmpdir )
2017-04-04 18:58:16 +02:00
if compare_result :
2017-04-23 11:33:51 +02:00
logging . debug ( ' removing %s ' , unsigned_apk )
os . remove ( unsigned_apk )
logging . debug ( ' removing %s ' , of )
os . remove ( of )
2017-04-04 18:58:16 +02:00
compare_result = compare_result . split ( ' \n ' )
line_count = len ( compare_result )
compare_result = compare_result [ : 299 ]
if line_count > len ( compare_result ) :
line_difference = \
line_count - len ( compare_result )
compare_result . append ( ' %d more lines ... ' %
line_difference )
compare_result = ' \n ' . join ( compare_result )
raise FDroidException ( ' compared built binary '
' to supplied reference '
' binary but failed ' ,
compare_result )
else :
logging . info ( ' compared built binary to '
' supplied reference binary '
' successfully ' )
2012-01-06 13:21:06 +01:00
build_succeeded . append ( app )
2013-05-03 08:55:40 +02:00
wikilog = " Build succeeded "
2017-04-04 18:58:16 +02:00
2012-01-02 15:09:20 +01:00
except VCSException as vcse :
2014-07-02 15:30:05 +02:00
reason = str ( vcse ) . split ( ' \n ' , 1 ) [ 0 ] if options . verbose else str ( vcse )
logging . error ( " VCS error while building app %s : %s " % (
2014-08-16 12:46:02 +02:00
appid , reason ) )
2012-01-08 19:27:54 +01:00
if options . stop :
2017-09-25 16:09:57 +02:00
logging . debug ( " Error encoutered, stopping by user request. " )
2018-11-14 14:27:32 +01:00
common . force_exit ( 1 )
2020-02-19 20:18:18 +01:00
add_failed_builds_entry ( failed_builds , appid , build , vcse )
2013-05-03 08:55:40 +02:00
wikilog = str ( vcse )
2015-11-14 13:05:37 +01:00
except FDroidException as e :
with open ( os . path . join ( log_dir , appid + ' .log ' ) , ' a+ ' ) as f :
2016-11-07 15:36:16 +01:00
f . write ( ' \n \n ============================================================ \n ' )
f . write ( ' versionCode: %s \n versionName: %s \n commit: %s \n ' %
2016-11-23 17:52:04 +01:00
( build . versionCode , build . versionName , build . commit ) )
2016-11-07 15:36:16 +01:00
f . write ( ' Build completed at '
2018-01-17 14:39:54 +01:00
+ common . get_wiki_timestamp ( ) + ' \n ' )
2016-11-07 15:36:16 +01:00
f . write ( ' \n ' + tools_version_log + ' \n ' )
2015-11-14 13:05:37 +01:00
f . write ( str ( e ) )
logging . error ( " Could not build app %s : %s " % ( appid , e ) )
if options . stop :
2017-09-25 16:09:57 +02:00
logging . debug ( " Error encoutered, stopping by user request. " )
2018-11-14 14:27:32 +01:00
common . force_exit ( 1 )
2020-02-19 20:18:18 +01:00
add_failed_builds_entry ( failed_builds , appid , build , e )
2015-11-14 13:05:37 +01:00
wikilog = e . get_wikitext ( )
2012-01-02 15:09:20 +01:00
except Exception as e :
2014-07-02 15:30:05 +02:00
logging . error ( " Could not build app %s due to unknown error: %s " % (
2014-08-16 12:46:02 +02:00
appid , traceback . format_exc ( ) ) )
2012-01-08 19:27:54 +01:00
if options . stop :
2017-09-25 16:09:57 +02:00
logging . debug ( " Error encoutered, stopping by user request. " )
2018-11-14 14:27:32 +01:00
common . force_exit ( 1 )
2020-02-19 20:18:18 +01:00
add_failed_builds_entry ( failed_builds , appid , build , e )
2013-05-03 08:55:40 +02:00
wikilog = str ( e )
2013-05-04 11:10:12 +02:00
if options . wiki and wikilog :
2013-05-03 08:55:40 +02:00
try :
2014-07-17 15:05:16 +02:00
# Write a page with the last build log for this version code
2016-11-23 17:52:04 +01:00
lastbuildpage = appid + ' /lastbuild_ ' + build . versionCode
2014-07-17 15:05:16 +02:00
newpage = site . Pages [ lastbuildpage ]
2017-01-20 12:10:35 +01:00
with open ( os . path . join ( ' tmp ' , ' fdroidserverid ' ) ) as fp :
2017-02-09 16:26:57 +01:00
fdroidserverid = fp . read ( ) . rstrip ( )
2018-01-22 14:00:16 +01:00
txt = " * build session started at " + common . get_wiki_timestamp ( start_timestamp ) + ' \n ' \
2018-01-03 12:16:20 +01:00
+ " * this build started at " + build_starttime + ' \n ' \
2018-01-17 14:39:54 +01:00
+ " * this build completed at " + common . get_wiki_timestamp ( ) + ' \n ' \
2018-03-05 21:47:19 +01:00
+ common . get_git_describe_link ( ) \
2017-01-20 12:10:35 +01:00
+ ' * fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/ ' \
2017-02-09 23:49:42 +01:00
+ fdroidserverid + ' ' + fdroidserverid + ' ] \n \n '
2018-01-03 13:58:06 +01:00
if buildserverid :
2017-02-09 23:49:42 +01:00
txt + = ' * buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/ ' \
+ buildserverid + ' ' + buildserverid + ' ] \n \n '
txt + = tools_version_log + ' \n \n '
txt + = ' == Build Log == \n \n ' + wikilog
2014-01-16 11:17:22 +01:00
newpage . save ( txt , summary = ' Build log ' )
2014-07-17 15:05:16 +02:00
# Redirect from /lastbuild to the most recent build log
2014-08-16 12:46:02 +02:00
newpage = site . Pages [ appid + ' /lastbuild ' ]
2014-07-17 15:05:16 +02:00
newpage . save ( ' #REDIRECT [[ ' + lastbuildpage + ' ]] ' , summary = ' Update redirect ' )
2017-03-22 14:01:32 +01:00
except Exception as e :
logging . error ( " Error while attempting to publish build log: %s " % e )
2012-01-03 08:43:14 +01:00
2018-01-15 01:03:47 +01:00
if timer :
timer . cancel ( ) # kill the watchdog timer
2018-01-19 22:35:06 +01:00
if max_build_time_reached :
2020-02-18 23:16:18 +01:00
status_output [ ' maxBuildTimeReached ' ] = True
2018-01-19 22:35:06 +01:00
logging . info ( " Stopping after global build timeout... " )
break
2012-02-26 15:09:25 +01:00
for app in build_succeeded :
2015-11-28 13:09:47 +01:00
logging . info ( " success: %s " % ( app . id ) )
2012-02-26 15:09:25 +01:00
2013-10-23 19:53:33 +02:00
if not options . verbose :
2020-02-19 20:18:18 +01:00
for fb in failed_builds :
logging . info ( ' Build for app {} : {} failed: \n {} ' . format ( * fb ) )
2012-02-26 15:09:25 +01:00
2017-09-13 18:03:57 +02:00
logging . info ( _ ( " Finished " ) )
2012-02-26 15:09:25 +01:00
if len ( build_succeeded ) > 0 :
2017-09-13 17:33:57 +02:00
logging . info ( ngettext ( " {} build succeeded " ,
" {} builds succeeded " , len ( build_succeeded ) ) . format ( len ( build_succeeded ) ) )
2020-02-19 20:18:18 +01:00
if len ( failed_builds ) > 0 :
2017-09-13 17:33:57 +02:00
logging . info ( ngettext ( " {} build failed " ,
2020-02-19 20:18:18 +01:00
" {} builds failed " , len ( failed_builds ) ) . format ( len ( failed_builds ) ) )
2011-01-04 22:44:14 +01:00
2018-01-22 14:00:16 +01:00
if options . wiki :
wiki_page_path = ' build_ ' + time . strftime ( ' %s ' , start_timestamp )
newpage = site . Pages [ wiki_page_path ]
txt = ' '
txt + = " * command line: <code> %s </code> \n " % ' ' . join ( sys . argv )
txt + = " * started at %s \n " % common . get_wiki_timestamp ( start_timestamp )
txt + = " * completed at %s \n " % common . get_wiki_timestamp ( )
if buildserverid :
txt + = ( ' * buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/ {id} {id} ] \n '
. format ( id = buildserverid ) )
if fdroidserverid :
txt + = ( ' * fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/ {id} {id} ] \n '
. format ( id = fdroidserverid ) )
if os . cpu_count ( ) :
txt + = " * host processors: %d \n " % os . cpu_count ( )
if os . path . isfile ( ' /proc/meminfo ' ) and os . access ( ' /proc/meminfo ' , os . R_OK ) :
with open ( ' /proc/meminfo ' ) as fp :
for line in fp :
m = re . search ( r ' MemTotal: \ s*([0-9].*) ' , line )
if m :
txt + = " * host RAM: %s \n " % m . group ( 1 )
break
2019-02-07 19:48:03 +01:00
fdroid_path = os . path . realpath ( os . path . join ( os . path . dirname ( __file__ ) , ' .. ' ) )
buildserver_config = os . path . join ( fdroid_path , ' makebuildserver.config.py ' )
if os . path . isfile ( buildserver_config ) and os . access ( buildserver_config , os . R_OK ) :
with open ( buildserver_config ) as configfile :
for line in configfile :
m = re . search ( r ' cpus \ s*= \ s*([0-9].*) ' , line )
if m :
txt + = " * guest processors: %s \n " % m . group ( 1 )
m = re . search ( r ' memory \ s*= \ s*([0-9].*) ' , line )
if m :
txt + = " * guest RAM: %s MB \n " % m . group ( 1 )
2018-01-22 14:00:16 +01:00
txt + = " * successful builds: %d \n " % len ( build_succeeded )
2020-02-19 20:18:18 +01:00
txt + = " * failed builds: %d \n " % len ( failed_builds )
2018-01-22 14:00:16 +01:00
txt + = " \n \n "
newpage . save ( txt , summary = ' Run log ' )
newpage = site . Pages [ ' build ' ]
newpage . save ( ' #REDIRECT [[ ' + wiki_page_path + ' ]] ' , summary = ' Update redirect ' )
2020-02-20 09:02:39 +01:00
if not options . onserver :
common . write_status_json ( status_output )
2020-02-18 23:16:18 +01:00
2017-12-02 13:24:13 +01:00
# hack to ensure this exits, even is some threads are still running
2018-11-14 14:27:32 +01:00
common . force_exit ( )
2012-01-03 17:02:28 +01:00
2016-11-15 21:55:06 +01:00
2012-02-26 15:09:25 +01:00
if __name__ == " __main__ " :
main ( )