Initial snapshot support. Building now requires KVM too

Makefile

@@ -1,5 +1,5 @@
 SHELL := /usr/bin/env bash
-VERSIONS = android-16 android-17 android-18 android-19 android-21 android-22 android-23 android-24 android-25 android-26
+VERSIONS = android-26
 
 generate:
 	for version in $(VERSIONS); do \
@@ -8,6 +8,9 @@ generate:
 		sed "s/{{ platform }}/$$version/g" templates/config.ini > build/$$version/config.ini ; \
 		sed "s/{{ platform }}/$$version/g" templates/start.sh > build/$$version/start.sh ; \
 		sed "s/{{ platform }}/$$version/g" templates/Makefile > build/$$version/Makefile ; \
+		sed "s/{{ platform }}/$$version/g" templates/snapshot.sh > build/$$version/snapshot.sh ; \
+		sed "s/{{ platform }}/$$version/g" templates/snapshot.expect > build/$$version/snapshot.expect ; \
+		sed "s/{{ platform }}/$$version/g" templates/take_snapshot.sh > build/$$version/take_snapshot.sh ; \
 		cp base/* ./build/$$version ; \
 	done
 

templates/Dockerfile

@@ -1,62 +1,60 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 MAINTAINER Anton Malinskiy "anton@malinskiy.com"
 
 # Set up insecure default key
-ADD adbkey adbkey.pub adb_usb.ini /root/.android/
+COPY adbkey adbkey.pub adb_usb.ini /root/.android/
 
-ENV LINK_ANDROID_SDK=https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip \
+ENV LINK_ANDROID_SDK=https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip \
     LANG=en_US.UTF-8 \
     LANGUAGE=en_US:en \
     LC_ALL=en_US.UTF-8 \
     ANDROID_HOME=/opt/android-sdk-linux \
     PATH="$PATH:/opt/android-sdk-linux/tools:/opt/android-sdk-linux/platform-tools:/opt/android-sdk-linux/tools/bin:/opt/android-sdk-linux/emulator"
 
-
 RUN dpkg --add-architecture i386 && \
-    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt xenial main restricted universe multiverse" > /etc/apt/sources.list && \
+    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt bionic main restricted universe multiverse" > /etc/apt/sources.list && \
-    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt xenial-updates main restricted universe multiverse" >> /etc/apt/sources.list && \
+    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-updates main restricted universe multiverse" >> /etc/apt/sources.list && \
-    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt xenial-security main restricted universe multiverse" >> /etc/apt/sources.list && \
+    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-security main restricted universe multiverse" >> /etc/apt/sources.list && \
-    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt xenial-backports main restricted universe multiverse" >> /etc/apt/sources.list && \
+    echo "deb mirror://mirrors.ubuntu.com/mirrors.txt bionic-backports main restricted universe multiverse" >> /etc/apt/sources.list && \
     apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get install -yq software-properties-common libstdc++6:i386 zlib1g:i386 libncurses5:i386 \
                         locales ca-certificates apt-transport-https curl unzip redir iproute2 \
-                        openjdk-8-jdk xvfb x11vnc fluxbox nano libpulse0 \
+                        openjdk-8-jdk xvfb x11vnc fluxbox nano libpulse0 telnet expect\
                         --no-install-recommends && \
     locale-gen en_US.UTF-8 && \
-
     # Install Android SDK
     curl -L $LINK_ANDROID_SDK > /tmp/android-sdk-linux.zip && \
     unzip -q /tmp/android-sdk-linux.zip -d /opt/android-sdk-linux/ && \
     rm /tmp/android-sdk-linux.zip && \
-
     # Customized steps per specific platform
     yes | sdkmanager --no_https --licenses && \
-    sdkmanager --no_https emulator tools platform-tools "platforms;{{ platform }}" "system-images;{{ platform }};google_apis;x86" --verbose && \
+    sdkmanager emulator tools platform-tools "platforms;{{ platform }}" "system-images;{{ platform }};google_apis;x86" --verbose && \
     echo no | avdmanager create avd -n "x86" --package "system-images;{{ platform }};google_apis;x86" --tag google_apis && \
-
     # Unfilter devices (now local because CI downloads from github are unstable)
     # curl -o /root/.android/adb_usb.ini https://raw.githubusercontent.com/apkudo/adbusbini/master/adb_usb.ini && \
-
-    # Optimize size of the image
-    rm /root/.android/avd/x86.avd/userdata.img && \
-    tar -czvf /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.img.tar.gz /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.img && \
-    rm /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.img && \
-    tar -czvf /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.img.tar.gz /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.img && \
-    rm /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.img && \
     DEBIAN_FRONTEND=noninteractive apt-get purge -yq unzip openjdk-8-jdk && \
-
+    # Convert large partitions to qcow2 to save space
+    qemu-img convert -O qcow2 -c /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.img /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.qcow2 && \
+    mv /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.qcow2 /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.img && \
+    qemu-img convert -O qcow2 -c /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.img /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.qcow2 && \
+    mv /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.qcow2 /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.img && \
+    qemu-img convert -O qcow2 -c /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/vendor.img /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/vendor.qcow2 && \
+    mv /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/vendor.qcow2 /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/vendor.img && \
+    qemu-img convert -O qcow2 -c /root/.android/avd/x86.avd/userdata.img /root/.android/avd/x86.avd/userdata.qcow2 && \
+    mv /root/.android/avd/x86.avd/userdata.qcow2 /root/.android/avd/x86.avd/userdata.img && \
+    # Clean up
     apt-get -yq autoremove && \
     apt-get clean && \
     apt-get autoclean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-ADD config.ini /root/.android/avd/x86.avd/config.ini
+COPY config.ini /root/.android/avd/x86.avd/config.ini
 
 # Expose adb
-EXPOSE 5037 5554 5555
+EXPOSE 5037 5554 5555 5900
 
 # Add script
-ADD start.sh /start.sh
+COPY start.sh /start.sh
 RUN chmod +x /start.sh
-CMD /start.sh
+CMD ["/start.sh"]

templates/config.ini

@@ -16,8 +16,8 @@ hw.device.hash2=MD5:1be89bc42ec9644d4b77968b23474980
 hw.device.manufacturer=Google
 hw.device.name=Nexus 5X
 hw.gps=yes
-hw.gpu.enabled=yes
+hw.gpu.enabled=on
-hw.gpu.mode=swiftshader
+hw.gpu.mode=swiftshader_indirect
 hw.initialOrientation=Portrait
 hw.keyboard=no
 hw.lcd.density=160
@@ -36,4 +36,4 @@ skin.path=_no_skin
 skin.path.backup=_no_skin
 tag.display=Google APIs
 tag.id=google_apis
-vm.heapSize=512
+vm.heapSize=192

templates/snapshot.expect

@@ -0,0 +1,12 @@
+#!/usr/bin/expect -f
+set timeout -1
+set TOKEN [lindex $argv 0];
+spawn telnet localhost 5554
+expect "OK"
+send -- "auth $TOKEN\r"
+expect "OK"
+send -- "avd snapshot del default_boot\r"
+expect "OK"
+send -- "avd snapshot save default\r"
+expect "OK"
+send "exit\r"

templates/snapshot.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+function save {
+  local token=$(cat /root/.emulator_console_auth_token)
+  expect -f /snapshot.expect $token
+}
+
+function clean_up {
+    echo "Cleaning up"
+    rm /tmp/.X1-lock
+
+    kill $XVFB_PID
+    exit
+}
+
+echo "Starting emulator"
+trap clean_up SIGHUP SIGINT SIGTERM
+export DISPLAY=:1
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/android-sdk-linux/emulator/lib64/qt/lib:/opt/android-sdk-linux/emulator/lib64/libstdc++:/opt/android-sdk-linux/emulator/lib64:/opt/android-sdk-linux/emulator/lib64/gles_swiftshader
+Xvfb :1 +extension GLX +extension RANDR +extension RENDER +extension XFIXES -screen 0 1024x768x24 &
+XVFB_PID=$!
+
+cd /opt/android-sdk-linux/emulator
+LIBGL_DEBUG=verbose ./qemu/linux-x86_64/qemu-system-i386 -avd x86 -snapshot default -no-snapshot-save &
+EMULATOR_PID=$!
+
+adb wait-for-device
+
+boot_completed=`adb -e shell getprop sys.boot_completed 2>&1`
+timeout=0
+until [ "X${boot_completed:0:1}" = "X1" ]; do
+    sleep 1
+    boot_completed=`adb shell getprop sys.boot_completed 2>&1 | head -n 1`
+    echo "Read boot_completed property: <$boot_completed>"
+    let "timeout += 1"
+    if [ $timeout -gt 300 ]; then
+         echo "Failed to start emulator"
+         exit 1
+    fi
+done
+
+sleep 5
+
+save
+adb emu kill
+
+# Doesn't work: triggers cold boot
+# qemu-img convert -O qcow2 -c /root/.android/avd/x86.avd/userdata-qemu.img /root/.android/avd/x86.avd/userdata-qemu.img_qcow2
+# mv /root/.android/avd/x86.avd/userdata-qemu.img_qcow2 /root/.android/avd/x86.avd/userdata-qemu.img
+
+# Moving adb binary away so that stopping adb server with delay will release the emulator and will make it available for external connections
+mv /opt/android-sdk-linux/platform-tools/adb /opt/android-sdk-linux/platform-tools/_adb
+
+echo "Great Scott!"
+clean_up

templates/start.sh

@@ -18,7 +18,7 @@ then
 fi
 if [ -z "$emulator_opts" ]
 then
-  emulator_opts="-screen multi-touch -no-boot-anim -noaudio -nojni -wipe-data -netfast -verbose -camera-back none -camera-front none -skip-adb-auth"
+  emulator_opts="-screen multi-touch -no-boot-anim -noaudio -nojni -netfast -verbose -camera-back none -camera-front none -skip-adb-auth -snapshot default -no-snapshot-save"
 fi
 
 # Detect ip and forward ADB ports outside to outside interface
@@ -27,23 +27,30 @@ redir --laddr=$ip --lport=$adb_server_port --caddr=127.0.0.1 --cport=$adb_server
 redir --laddr=$ip --lport=$console_port --caddr=127.0.0.1 --cport=$console_port &
 redir --laddr=$ip --lport=$adb_port --caddr=127.0.0.1 --cport=$adb_port &
 
-# Moving adb binary away so that stopping adb server with delay will release the emulator and will make it available for external connections
+function clean_up {
-mv /opt/android-sdk-linux/platform-tools/adb /opt/android-sdk-linux/platform-tools/_adb
+    echo "Cleaning up"
-sleep 30 && _adb kill-server &
+    rm /tmp/.X1-lock
 
+    kill $XVFB_PID
+    kill $FLUXBOX_PID
+    kill $VNC_PID
+    exit
+}
+
+trap clean_up SIGHUP SIGINT SIGTERM
 export DISPLAY=:1
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/android-sdk-linux/emulator/lib64/qt/lib:/opt/android-sdk-linux/emulator/lib64/libstdc++:/opt/android-sdk-linux/emulator/lib64:/opt/android-sdk-linux/emulator/lib64/gles_swiftshader
 Xvfb :1 +extension GLX +extension RANDR +extension RENDER +extension XFIXES -screen 0 1024x768x24 &
-fluxbox -display ":1.0" &
+XVFB_PID=$!
-x11vnc -display :1 -nopw -forever &
+sleep 1 && fluxbox -display ":1.0" &
+FLUXBOX_PID=$!
+sleep 2 && x11vnc -display :1 -nopw -forever &
+VNC_PID=$!
 
 # Set up and run emulator
 # qemu references bios by relative path
 cd /opt/android-sdk-linux/emulator
 
-tar -xvf /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/userdata.img.tar.gz --directory /
-tar -xvf /opt/android-sdk-linux/system-images/{{ platform }}/google_apis/x86/system.img.tar.gz --directory /
-
 CONFIG="/root/.android/avd/x86.avd/config.ini"
 CONFIGTMP=${CONFIG}.tmp
 
@@ -60,4 +67,6 @@ then
   done
 fi
 
+echo "emulator_opts: $emulator_opts"
+
 LIBGL_DEBUG=verbose ./qemu/linux-x86_64/qemu-system-i386 -avd x86 -ports $console_port,$adb_port $emulator_opts -qemu $QEMU_OPTS

templates/take_snapshot.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -ex
+
+docker rm -f emulator || true
+
+docker run -d -t --name emulator --rm --privileged -v /dev/kvm:/dev/kvm -e ANDROID_ARCH="x86" agoda/docker-emulator-{{ platform }} bash
+
+docker cp snapshot.sh emulator:/snapshot.sh
+docker cp snapshot.expect emulator:/snapshot.expect
+docker exec -t emulator bash -c "bash /snapshot.sh; exit"
+echo "Creating new image"
+docker commit -m "Snapshot!" --change "CMD [\"/start.sh\"]" emulator agoda/docker-emulator-{{ platform }}-snapshot
+docker rm -f emulator