Enhance screenshot handling

Signed-off-by: tobiasKaminsky <tobias@kaminsky.me>
2021-07-15 09:00:38 +02:00 · 2021-07-15 09:00:38 +02:00 · a57f8eb989
parent 545197d68c
commit a57f8eb989
26 changed files with 233 additions and 35 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -214,3 +214,57 @@ trigger:
  event:
    - push
    - pull_request
+---
+kind: pipeline
+type: docker
+name: allScreenshots
+
+steps:
+  - name: runAllScreenshots
+    image: nextcloudci/android8:android-61
+    privileged: true
+    environment:
+      GIT_USERNAME:
+        from_secret: GIT_USERNAME
+      GIT_TOKEN:
+        from_secret: GIT_TOKEN
+      LOG_USERNAME:
+        from_secret: LOG_USERNAME
+      LOG_PASSWORD:
+        from_secret: LOG_PASSWORD
+    commands:
+      - emulator -avd android -no-snapshot -gpu swiftshader_indirect -no-window -no-audio -skin 500x833 &
+      - sed -i s'#<bool name="is_beta">false</bool>#<bool name="is_beta">true</bool>#'g src/main/res/values/setup.xml
+      - sed -i s'#showOnlyFailingTestsInReports = ciBuild#showOnlyFailingTestsInReports = false#' build.gradle
+      - scripts/wait_for_emulator.sh
+      - scripts/runAllScreenshotCombinations noCI false
+      - scripts/screenshotSummary.sh
+  - name: notify
+    image: drillster/drone-email
+    settings:
+      port: 587
+      from: nextcloud-drone@kaminsky.me
+      recipients_only: true
+      username:
+        from_secret: EMAIL_USERNAME
+      password:
+        from_secret: EMAIL_PASSWORD
+      recipients:
+        from_secret: EMAIL_RECIPIENTS
+      host:
+        from_secret: EMAIL_HOST
+    when:
+      event:
+        - push
+      status:
+        - failure
+      branch:
+        - master
+        - stable-*
+trigger:
+  branch:
+    - master
+  event:
+    - cron
+  cron:
+    - nightly
--- a/.github/workflows/screenShotTest.yml
+++ b/.github/workflows/screenShotTest.yml
@ -10,8 +10,8 @@ jobs:
        strategy:
            fail-fast: false
            matrix:
-                scheme: [ Dark, Light ]
-                color: [ blue, white, black ]
+                scheme: [ Light ]
+                color: [ blue ]
        steps:
            -   uses: actions/checkout@v2
            -   name: set up JDK 1.8
--- a/screenshots/gplay/debug/com.nextcloud.client.AuthenticatorActivityIT_login.png
+++ b/screenshots/gplay/debug/com.nextcloud.client.AuthenticatorActivityIT_login.png
--- a/screenshots/gplay/debug/com.nextcloud.client.FileDisplayActivityScreenshotIT_open.png
+++ b/screenshots/gplay/debug/com.nextcloud.client.FileDisplayActivityScreenshotIT_open.png
--- a/screenshots/gplay/debug/com.nextcloud.client.NotificationsActivityIT_openDrawer.png
+++ b/screenshots/gplay/debug/com.nextcloud.client.NotificationsActivityIT_openDrawer.png
--- a/screenshots/gplay/debug/com.nextcloud.client.SettingsActivityIT_open.png
+++ b/screenshots/gplay/debug/com.nextcloud.client.SettingsActivityIT_open.png
--- a/screenshots/gplay/debug/com.nextcloud.client.etm.EtmActivityTest_accounts.png
+++ b/screenshots/gplay/debug/com.nextcloud.client.etm.EtmActivityTest_accounts.png
--- a/screenshots/gplay/debug/com.nextcloud.client.etm.EtmActivityTest_overview.png
+++ b/screenshots/gplay/debug/com.nextcloud.client.etm.EtmActivityTest_overview.png
--- a/screenshots/gplay/debug/com.nextcloud.client.etm.EtmActivityTest_preferences.png
+++ b/screenshots/gplay/debug/com.nextcloud.client.etm.EtmActivityTest_preferences.png
--- a/screenshots/gplay/debug/com.owncloud.android.ui.activity.FolderPickerActivityIT_open.png
+++ b/screenshots/gplay/debug/com.owncloud.android.ui.activity.FolderPickerActivityIT_open.png
--- a/screenshots/gplay/debug/com.owncloud.android.ui.activity.ManageAccountsActivityIT_userInfoDetail.png
+++ b/screenshots/gplay/debug/com.owncloud.android.ui.activity.ManageAccountsActivityIT_userInfoDetail.png
--- a/screenshots/gplay/debug/com.owncloud.android.ui.activity.NotificationsActivityIT_loading.png
+++ b/screenshots/gplay/debug/com.owncloud.android.ui.activity.NotificationsActivityIT_loading.png
--- a/screenshots/gplay/debug/com.owncloud.android.ui.activity.PassCodeActivityIT_check.png
+++ b/screenshots/gplay/debug/com.owncloud.android.ui.activity.PassCodeActivityIT_check.png
--- a/screenshots/gplay/debug/com.owncloud.android.ui.activity.PassCodeActivityIT_delete.png
+++ b/screenshots/gplay/debug/com.owncloud.android.ui.activity.PassCodeActivityIT_delete.png
--- a/screenshots/gplay/debug/com.owncloud.android.ui.activity.PassCodeActivityIT_request.png
+++ b/screenshots/gplay/debug/com.owncloud.android.ui.activity.PassCodeActivityIT_request.png
--- a/scripts/generateScreenshotOverview.sh
+++ b/scripts/generateScreenshotOverview.sh
@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Nextcloud Android client application
+#
+# @author Tobias Kaminsky
+# Copyright (C) 2021 Tobias Kaminsky
+# Copyright (C) 2021 Nextcloud GmbH
+#  
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#  
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#  
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+error=0
+total=0
+
+cp scripts/screenshotCombinations scripts/screenshotCombinations_
+grep -v "#" scripts/screenshotCombinations_ > scripts/screenshotCombinations
+rm scripts/screenshotCombinations_
+
+echo '<!DOCTYPE html>
+<html lang="de">
+<head>
+<meta charset="utf-8"/>
+</head>'
+
+echo "<table>"
+echo "<tr><td style='width:150px'>Original</td>"
+while read line; do
+    echo "<td style='width:150px'>$line</td>"
+done < scripts/screenshotCombinations
+echo "</tr>"
+
+#for image in ./build/reports/shot/verification/images/*.png ; do
+for image in $(/bin/ls -1 ./screenshots/gplay/debug/*.png | grep -v _dark_ | grep -v _light_) ; do
+    cp $image build/screenshotSummary/images/
+    
+    echo "<tr style='height:200px'>"
+    echo "<td><a target='_blank' href=\"images/$(basename $image)\"><img width=100px src=\"images/$(basename $image)\"/></a></td>"
+
+    while read line; do
+        echo "<td>"
+        
+        mode=$(echo $line | cut -d" " -f1)
+        color=$(echo $line | cut -d" " -f2)
+        total=$((total + 1))
+        
+        if [ $mode = "light" -a $color = "blue" ]; then
+            name=$(basename $image)
+        else
+            name=$(basename $image| sed s"/\.png/_${mode}_$color\.png/")
+        fi
+        
+        # if image does not exist
+        if [ ! -e ./build/reports/shot/verification/images/$name ]; then
+            echo "<span style='color: red'>✘</span>"
+            error=$((error + 1))
+        elif [ -e ./build/reports/shot/verification/images/diff_$name ]; then
+            # file with "diff_" prefix
+            cp ./build/reports/shot/verification/images/diff_$name build/screenshotSummary/images/
+            echo "<a target='_blank' href=\"images/diff_$name\"><img width=100px src=\"images/diff_$name\"/></a>"
+            error=$((error + 1))
+        else 
+            echo "✔"
+        fi
+        
+        echo "</td>"
+    done < scripts/screenshotCombinations
+    
+    echo "</tr>"
+done
+
+echo "</table>"
+
+echo "ERROR: $error / $total"
+echo "</html>"
--- a/scripts/runAllScreenshotCombinations
+++ b/scripts/runAllScreenshotCombinations
@ -16,12 +16,12 @@ fi
 classMethod=$3

 resultCode=0
-while read line
+grep -v "#" scripts/screenshotCombinations | while read line 
 do
    darkMode=$(echo "$line" | cut -d" " -f1)
    color=$(echo "$line" | cut -d" " -f2)

-    echo "Run $color on $darkMode mode"
+    echo -n "Run $color on $darkMode mode"

    if [[ $1 = "noCI" ]]; then
        ./gradlew --console plain gplayDebugExecuteScreenshotTests \
@ -32,7 +32,9 @@ do
        -Pandroid.testInstrumentationRunnerArguments.DARKMODE="$darkMode" \
        $classMethod </dev/null > /dev/null
        if [[ $? -ne 0 ]]; then
-            exit
+            echo " failed!"
+        else 
+            echo
        fi
    else
        ./gradlew --console plain gplayDebugExecuteScreenshotTests \
@ -45,7 +47,7 @@ do
        || resultCode=1 && scripts/uploadReport.sh "$LOG_USERNAME" "$LOG_PASSWORD" "$4" \
        "$1-$darkMode-$color" "Screenshot" "$4" "$GIT_TOKEN"
    fi
-done < scripts/screenshotCombinations
+done

 sed -i s'#<bool name="is_beta">true</bool>#<bool name="is_beta">false</bool>#'g src/main/res/values/setup.xml

--- a/scripts/screenshotCombinations
+++ b/scripts/screenshotCombinations
@ -1,4 +1,10 @@
 light blue
-dark blue
-light white
-dark white
+#dark blue
+#light white
+#dark white
+#light black
+#dark black
+#light red
+#dark red
+#light lightgreen
+dark lightgreen
--- a/scripts/screenshotSummary.sh
+++ b/scripts/screenshotSummary.sh
@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Nextcloud Android client application
+#
+# @author Tobias Kaminsky
+# Copyright (C) 2021 Tobias Kaminsky
+# Copyright (C) 2021 Nextcloud GmbH
+#  
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#  
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#  
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+mkdir -p build/screenshotSummary/images
+
+scripts/generateScreenshotOverview.sh > build/screenshotSummary/summary.html
+error=$?
+
+scripts/uploadScreenshotSummary.sh $LOG_USERNAME $LOG_PASSWORD
+
+exit $error
--- a/scripts/uploadScreenshotSummary.sh
+++ b/scripts/uploadScreenshotSummary.sh
@ -0,0 +1,17 @@
+#!/bin/bash -x
+
+#1: LOG_USERNAME
+#2: LOG_PASSWORD
+
+DAV_URL=https://nextcloud.kaminsky.me/remote.php/webdav/android-screenshot-summary/
+PUBLIC_URL=https://www.kaminsky.me/nc-dev/android-screenshot-summary
+USER=$1
+PASS=$2
+
+date=$(date +%F)
+echo "Uploaded screenshot summary to $PUBLIC_URL/$date/summary.html"
+
+cd build/screenshotSummary
+
+find . -type d -exec curl > /dev/null 2>&1 -u $USER:$PASS -X MKCOL $DAV_URL/$date/$(echo {} | sed s#\./##) \;
+find . -type f -exec curl > /dev/null 2>&1 -u $USER:$PASS -X PUT $DAV_URL/$date/$(echo {} | sed s#\./##) --upload-file {} \;
--- a/src/androidTest/java/com/nextcloud/client/FileDisplayActivityScreenshotIT.java
+++ b/src/androidTest/java/com/nextcloud/client/FileDisplayActivityScreenshotIT.java
@ -27,6 +27,7 @@ import android.Manifest;
 import com.owncloud.android.AbstractIT;
 import com.owncloud.android.R;
 import com.owncloud.android.ui.activity.FileDisplayActivity;
+import com.owncloud.android.ui.fragment.OCFileListFragment;
 import com.owncloud.android.utils.ScreenshotTest;

 import org.junit.Rule;
@ -39,6 +40,7 @@ import androidx.test.rule.GrantPermissionRule;

 import static androidx.test.espresso.Espresso.onView;
 import static androidx.test.espresso.matcher.ViewMatchers.withId;
+import static org.junit.Assert.assertNotNull;

 public class FileDisplayActivityScreenshotIT extends AbstractIT {
    @Rule public IntentsTestRule<FileDisplayActivity> activityRule = new IntentsTestRule<>(FileDisplayActivity.class,
@ -64,14 +66,16 @@ public class FileDisplayActivityScreenshotIT extends AbstractIT {
        screenshot(sut);
    }

-    @Test
-    @ScreenshotTest
+    //@Test
+    //@ScreenshotTest
    public void showMediaThenAllFiles() {
-        FileDisplayActivity sut = activityRule.launchActivity(null);
+        FileDisplayActivity fileDisplayActivity = activityRule.launchActivity(null);
+        OCFileListFragment sut = fileDisplayActivity.getListOfFilesFragment();
+        assertNotNull(sut);

-        sut.getListOfFilesFragment().setFabEnabled(false);
-        sut.getListOfFilesFragment().setEmptyListLoadingMessage();
-        sut.getListOfFilesFragment().setLoading(false);
+        sut.setFabEnabled(false);
+        sut.setEmptyListLoadingMessage();
+        sut.setLoading(false);

        // open drawer
        onView(withId(R.id.drawer_layout)).perform(DrawerActions.open());
@ -89,11 +93,11 @@ public class FileDisplayActivityScreenshotIT extends AbstractIT {

        // then compare screenshot
        shortSleep();
-        sut.getListOfFilesFragment().setFabEnabled(false);
-        sut.getListOfFilesFragment().setEmptyListLoadingMessage();
-        sut.getListOfFilesFragment().setLoading(false);
+        sut.setFabEnabled(false);
+        sut.setEmptyListLoadingMessage();
+        sut.setLoading(false);
        shortSleep();
-        screenshot(sut);
+        screenshot(fileDisplayActivity);
    }

    @Test
--- a/src/androidTest/java/com/nextcloud/client/etm/EtmActivityTest.kt
+++ b/src/androidTest/java/com/nextcloud/client/etm/EtmActivityTest.kt
@ -25,11 +25,9 @@ package com.nextcloud.client.etm
 import android.app.Activity
 import androidx.test.espresso.intent.rule.IntentsTestRule
 import androidx.test.internal.runner.junit4.statement.UiThreadStatement
+import com.nextcloud.client.preferences.AppPreferencesImpl
 import com.owncloud.android.AbstractIT
-import com.owncloud.android.lib.resources.status.OwnCloudVersion
 import com.owncloud.android.utils.ScreenshotTest
-import org.junit.Assume
-import org.junit.Before
 import org.junit.Rule
 import org.junit.Test

@ -37,16 +35,6 @@ class EtmActivityTest : AbstractIT() {
    @get:Rule
    var activityRule = IntentsTestRule(EtmActivity::class.java, true, false)

-    @Before
-    fun before() {
-        // tests only on NC 18
-        Assume.assumeTrue(
-            storageManager
-                .getCapability(account.name)
-                .version.compareTo(OwnCloudVersion.nextcloud_18) == 0
-        )
-    }
-
    @Test
    @ScreenshotTest
    fun overview() {
@ -62,7 +50,11 @@ class EtmActivityTest : AbstractIT() {
    fun preferences() {
        val sut: EtmActivity = activityRule.launchActivity(null)

-        UiThreadStatement.runOnUiThread { sut.vm.onPageSelected(0) }
+        UiThreadStatement.runOnUiThread {
+            val preferences = AppPreferencesImpl.fromContext(targetContext)
+            preferences.pushToken = "Push token"
+            sut.vm.onPageSelected(0)
+        }

        screenshot(sut)
    }
--- a/src/androidTest/java/com/owncloud/android/AbstractIT.java
+++ b/src/androidTest/java/com/owncloud/android/AbstractIT.java
@ -156,6 +156,10 @@ public abstract class AbstractIT {
                    colorHex = "#000000";
                    break;

+                case "lightgreen":
+                    colorHex = "#aaff00";
+                    break;
+
                default:
                    break;
            }
--- a/src/androidTest/java/com/owncloud/android/ui/activity/NotificationsActivityIT.kt
+++ b/src/androidTest/java/com/owncloud/android/ui/activity/NotificationsActivityIT.kt
@ -40,6 +40,10 @@ class NotificationsActivityIT : AbstractIT() {
    fun loading() {
        val sut: NotificationsActivity = activityRule.launchActivity(null)

+        waitForIdleSync()
+
+        sut.runOnUiThread { sut.setLoadingMessageEmpty() }
+
        screenshot(sut)
    }

--- a/src/main/java/com/owncloud/android/operations/RefreshFolderOperation.java
+++ b/src/main/java/com/owncloud/android/operations/RefreshFolderOperation.java
@ -290,7 +290,7 @@ public class RefreshFolderOperation extends RemoteOperation {
            } else {
                Log_OC.i(TAG, "Got display name: " + result.getResultData());
            }
-        } catch (AccountUtils.AccountNotFoundException e) {
+        } catch (AccountUtils.AccountNotFoundException | NullPointerException e) {
            Log_OC.e(this, "Error updating profile", e);
        }
    }
--- a/src/main/java/com/owncloud/android/ui/asynctasks/RetrieveStatusAsyncTask.java
+++ b/src/main/java/com/owncloud/android/ui/asynctasks/RetrieveStatusAsyncTask.java
@ -61,7 +61,7 @@ public class RetrieveStatusAsyncTask extends AsyncTask<Void, Void, Status> {
            } else {
                return new com.owncloud.android.lib.resources.users.Status(StatusType.OFFLINE, "", "", -1);
            }
-        } catch (ClientFactory.CreationException e) {
+        } catch (ClientFactory.CreationException | NullPointerException e) {
            return new com.owncloud.android.lib.resources.users.Status(StatusType.OFFLINE, "", "", -1);
        }
    }