Publish Vulpes Preview 15 sources for desktop, Android and Tundra

Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.

gaia/test_media/reference-workload/generateImages.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd $(dirname $0); pwd)
+
+if [ -z "$1" ]; then
+  echo Must provide number of iterations
+  exit 1
+fi
+
+if [ "$1" != "0" ]; then
+#  REMOTE_DIR="/sdcard/DCIM/100MZLLA"
+  REMOTE_DIR=
+  for dir in /sdcard /storage/sdcard /storage/sdcard0; do
+    if [ -n "$($ADB_REF shell "test -d $dir && echo found")" ]; then
+      REMOTE_DIR=$dir
+      break
+    fi
+  done
+
+  if [ -z "$REMOTE_DIR" ]; then
+    echo "Can't find remote dir" >&2
+    exit 1
+  fi
+
+  $ADB_REF push ${SCRIPT_DIR}/MasterGalleryImage.jpg ${REMOTE_DIR}/DCIM/100MZLLA/IMG_0001.jpg || exit 1
+
+  for i in `seq -f '%04g' 2 $1` ; do
+    FILENAME=IMG_$i.jpg
+    $ADB_REF shell "cat ${REMOTE_DIR}/DCIM/100MZLLA/IMG_0001.jpg > ${REMOTE_DIR}/DCIM/100MZLLA/${FILENAME}"
+  done
+fi

gaia/test_media/reference-workload/generateMusicFiles.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd $(dirname $0); pwd)
+
+if [ -z "$1" ]; then
+  echo "Must provide number of iterations"
+  exit 1
+fi
+
+if ! type mid3v2 > /dev/null 2>&1; then
+  echo "mid3v2 required to load music - install with |sudo apt-get install python-mutagen|"
+  echo "No music loaded"
+  exit 1
+fi
+
+ALBUM=1
+TRACK=1
+ARTIST=1
+SONG_NAME=MasterSong.mp3
+
+# remove all the ID3 tags from the song
+mid3v2 -D ${SCRIPT_DIR}/${SONG_NAME}
+
+# We start with 1 track per album, and increment that up to 10, and then cycle between 5 and 10
+TRACKS_PER_ALBUM=1
+# REMOTE_DIR="/sdcard/Music"
+REMOTE_DIR=
+for dir in /sdcard /storage/sdcard /storage/sdcard0; do
+  if [ -n "$($ADB_REF shell "test -d $dir && echo found")" ]; then
+    REMOTE_DIR=$dir
+    break
+  fi
+done
+
+if [ -z "$REMOTE_DIR" ]; then
+  echo "Can't find remote dir" >&2
+  exit 1
+fi
+
+for i in `seq -f '%04g' 2 $1` ; do
+  FILENAME=SONG_$i.mp3
+
+  mid3v2 -t "Song ${i}" -a "Artist ${ARTIST}" -A "Album ${ALBUM}" -T "${TRACK}" ${SCRIPT_DIR}/${SONG_NAME} || exit 1
+  $ADB_REF push ${SCRIPT_DIR}/${SONG_NAME} ${REMOTE_DIR}/Music/${FILENAME} || exit 1
+
+  let TRACK=TRACK+1
+  if [ ${TRACK} -gt ${TRACKS_PER_ALBUM} ]; then
+    let TRACK=1
+    let ALBUM=ALBUM+1
+    let TRACKS_PER_ALBUM=TRACKS_PER_ALBUM+1
+    if [ ${TRACKS_PER_ALBUM} -gt 10 ]; then
+      let TRACKS_PER_ALBUM=5
+      let ARTIST=ARTIST+1
+    fi
+  fi
+done
+
+# remove all the ID3 tags from the song (again)
+mid3v2 -D ${SCRIPT_DIR}/${SONG_NAME} || exit 1

gaia/test_media/reference-workload/generateVideos.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd $(dirname $0); pwd)
+
+if [ -z "$1" ]; then
+  echo Must provide number of iterations
+  exit 1
+fi
+
+if [ "$1" != "0" ]; then
+# REMOTE_DIR="/sdcard/Movies"
+  REMOTE_DIR=
+  for dir in /sdcard /storage/sdcard /storage/sdcard0; do
+    if [ -n "$($ADB_REF shell "test -d $dir && echo found")" ]; then
+      REMOTE_DIR=$dir
+      break
+    fi
+  done
+  if [ -z "$REMOTE_DIR" ]; then
+    echo "Can't find remote dir" >&2
+    exit 1
+  fi
+  $ADB_REF push ${SCRIPT_DIR}/MasterVideo.3gp ${REMOTE_DIR}/Movies/VID_0001.3gp || exit 1
+
+  for i in `seq -f '%04g' 2 $1` ; do
+    FILENAME=VID_$i.3gp
+    $ADB_REF shell "cat ${REMOTE_DIR}/Movies/VID_0001.3gp > ${REMOTE_DIR}/Movies/${FILENAME}"
+  done
+
+fi

gaia/test_media/reference-workload/makeReferenceWorkload.sh

@@ -0,0 +1,207 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$(cd $(dirname $0); pwd)
+
+json=$BUILD_CONFIG
+prop='ADB'
+
+function jsonval {
+  temp=`echo $json | sed 's/\\\\\//\//g' | sed 's/[{}]//g' | awk -v k="text" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' | sed 's/\"\:\"/\|/g' | sed 's/[\,]/ /g' | sed 's/\"//g' | grep -w $prop| cut -d":" -f2| sed -e 's/^ *//g' -e 's/ *$//g'`
+  echo ${temp##*|}
+}
+
+export ADB_REF=$(jsonval)
+
+if [ -z "$1" ]; then
+  echo "Must provide size parameter (light/medium/heavy/x-heavy)"
+  exit 1
+fi
+
+echo "Waiting for device to be connected..."
+$ADB_REF wait-for-device
+echo "Device connected"
+
+case $1 in
+
+  empty)
+    IMAGE_COUNT=0
+    MUSIC_COUNT=0
+    VIDEO_COUNT=0
+    CONTACT_COUNT=0
+    SMS_COUNT=0
+    DIALER_COUNT=0
+  ;;
+
+  light)
+    IMAGE_COUNT=20
+    MUSIC_COUNT=20
+    VIDEO_COUNT=5
+    CONTACT_COUNT=200
+    SMS_COUNT=200
+    DIALER_COUNT=50
+  ;;
+
+  medium)
+    IMAGE_COUNT=50
+    MUSIC_COUNT=50
+    VIDEO_COUNT=10
+    CONTACT_COUNT=500
+    SMS_COUNT=500
+    DIALER_COUNT=100
+  ;;
+
+  heavy)
+    IMAGE_COUNT=100
+    MUSIC_COUNT=100
+    VIDEO_COUNT=20
+    CONTACT_COUNT=1000
+    SMS_COUNT=1000
+    DIALER_COUNT=200
+  ;;
+
+  x-heavy)
+    IMAGE_COUNT=250
+    MUSIC_COUNT=250
+    VIDEO_COUNT=50
+    CONTACT_COUNT=2000
+    SMS_COUNT=2000
+    DIALER_COUNT=500
+  ;;
+
+  *)
+    echo "Size parameter must be one of (empty/light/medium/heavy/x-heavy)"
+    exit 1
+
+esac
+
+echo "Populate Databases - $1 Workload"
+
+$ADB_REF shell stop b2g
+APPS=${APPS:-${APP}}
+
+IDB_BASE=
+for dir in /data/local/storage/permanent /data/local/storage/persistent /data/local/indexedDB; do
+  if [ -n "$($ADB_REF shell "test -d $dir/chrome && echo found")" ]; then
+    IDB_BASE=$dir
+
+    if [ "$IDB_BASE" == "/data/local/storage/permanent" ]; then
+      IDB_DEFAULT_BASE=/data/local/storage/default
+    else
+      IDB_DEFAULT_BASE="$IDB_BASE"
+    fi
+
+    break
+  fi
+done
+if [ -z "$IDB_BASE" ]; then
+  echo "Can't find indexedDB base dir" >&2
+  exit 1
+fi
+echo "IndexedDB base dir: $IDB_BASE"
+IDB_PRESENT=$($ADB_REF shell "ls -l $IDB_BASE/chrome/" | grep '^d.*idb')
+if [ -z "$IDB_PRESENT" ]; then
+  echo "idb directory not present"
+  IDB_PATH=""
+else
+  echo "idb directory present"
+  IDB_PATH="/idb"
+fi
+
+if [ -z "$APPS" ]; then
+  APPS="gallery music video communications/contacts sms communications/dialer"
+fi
+
+# Convert comma-separate APPS to space-separated
+APPS=${APPS//,/ }
+
+SUMMARY="Summary:\n"
+
+for app in $APPS; do
+
+  LINE=
+  case $app in
+    communications/dialer)
+      echo "Starting dialer"
+      $ADB_REF pull /data/local/webapps/webapps.json $SCRIPT_DIR/webapps.json || exit 1
+      DIALER_INFO=$(python $SCRIPT_DIR/readJSON.py $SCRIPT_DIR/webapps.json "communications.*/localId")
+      IFS='/' read -a DIALER_PARTS <<< "$DIALER_INFO"
+      DIALER_DOMAIN=${DIALER_PARTS[0]}
+      DIALER_ID=${DIALER_PARTS[1]}
+      DIALER_DIR="$DIALER_ID+f+app+++$DIALER_DOMAIN"
+      rm -f $SCRIPT_DIR/webapps.json
+      if [ -z "$DIALER_ID" ]; then
+        echo "Unable to determine communications application ID - skipping dialer history..."
+        LINE=" Dialer History: skipped"
+      else
+        $ADB_REF push  $SCRIPT_DIR/dialerDb-$DIALER_COUNT.sqlite $IDB_DEFAULT_BASE/$DIALER_DIR$IDB_PATH/2584670174dsitanleecreR.sqlite || exit 1
+        LINE=" Dialer History: $(printf "%4d" $DIALER_COUNT)"
+      fi
+      ;;
+
+    gallery)
+      echo "Starting gallery"
+      $SCRIPT_DIR/generateImages.sh $IMAGE_COUNT || exit 1
+      LINE=" Gallery:        $(printf "%4d" $IMAGE_COUNT)"
+      ;;
+
+    music)
+      echo "Starting music"
+      $SCRIPT_DIR/generateMusicFiles.sh $MUSIC_COUNT || exit 1
+      LINE=" Music:          $(printf "%4d" $MUSIC_COUNT)"
+      ;;
+
+    video)
+      echo "Starting video"
+      $SCRIPT_DIR/generateVideos.sh $VIDEO_COUNT || exit 1
+      LINE=" Videos:         $(printf "%4d" $VIDEO_COUNT)"
+      ;;
+
+    communications/contacts)
+      echo "Starting contacts"
+      # Accross versions, we can have various names for the attachment
+      # directory, so let's delete all of them
+      $ADB_REF shell "rm -r $IDB_BASE/chrome$IDB_PATH/*csotncta*"
+      $ADB_REF push  $SCRIPT_DIR/contactsDb-$CONTACT_COUNT.sqlite $IDB_BASE/chrome$IDB_PATH/3406066227csotncta.sqlite || exit 1
+      ATTACHMENT_DIR=$SCRIPT_DIR/contactsDb-$CONTACT_COUNT
+      tar -xvzf $SCRIPT_DIR/ContactPictures-$CONTACT_COUNT.tar.gz -C $SCRIPT_DIR
+      # In recent builds, the directory will be automatically renamed to
+      # 3406066227csotncta.files when B2G starts up.
+      $ADB_REF push  $SCRIPT_DIR/contactsDb-$CONTACT_COUNT/ $IDB_BASE/chrome$IDB_PATH/3406066227csotncta/ || exit 1
+      rm -rf $ATTACHMENT_DIR/
+      LINE=" Contacts:       $(printf "%4d" $CONTACT_COUNT)"
+      ;;
+
+    sms)
+      echo "Starting sms"
+      # Accross versions, we can have various names for the attachment
+      # directory, so let's delete all of them
+      $ADB_REF shell "rm -r $IDB_BASE/chrome$IDB_PATH/*ssm*"
+      $ADB_REF push  $SCRIPT_DIR/smsDb-$SMS_COUNT.sqlite $IDB_BASE/chrome$IDB_PATH/226660312ssm.sqlite || exit 1
+      ATTACHMENT_DIR=$SCRIPT_DIR/smsDb-$SMS_COUNT
+      tar -xvzf $SCRIPT_DIR/Attachments-$SMS_COUNT.tar.gz -C $SCRIPT_DIR
+      # In recent builds, the directory will be automatically renamed to
+      # 226660312ssm.files when B2G starts up.
+      $ADB_REF push  $SCRIPT_DIR/smsDb-$SMS_COUNT/ $IDB_BASE/chrome$IDB_PATH/226660312ssm/ || exit 1
+      rm -rf $ATTACHMENT_DIR/
+      LINE=" Sms Messages:   $(printf "%4d" $SMS_COUNT)"
+      ;;
+
+    *)
+      echo "APPS includes unknown application name ($app) - ignoring..."
+      LINE=" $app: unknown application name"
+  esac
+
+  if [ -n "$LINE" ]; then
+    SUMMARY="$SUMMARY$LINE\n"
+  fi
+
+done
+
+echo ""
+echo -e "$SUMMARY"
+
+$ADB_REF shell start b2g
+
+echo "Done"

gaia/test_media/reference-workload/readJSON.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+#
+# This script parses json files, and extracts a value from a provided key
+# The value is printed to stdout
+
+import json
+import os
+import sys
+
+json_data=open(sys.argv[1])
+data = json.load(json_data)
+json_data.close()
+
+keyString = sys.argv[2]
+keys = keyString.split('/')
+
+result = data
+matchedKey = ""
+for key in keys:
+  if key.endswith("*"):
+    matchKey =  key.rstrip("*")
+    for eachKey, eachValue in data.items():
+      if eachKey.startswith(matchKey):
+        result = result[eachKey]
+        matchedKey = eachKey
+        break
+  else:
+    result = result[key]
+
+if matchedKey == "":
+  print result
+else:
+  print matchedKey + "/" + str(result)

gaia/test_media/reference-workload/readme

@@ -0,0 +1,36 @@
+
+Reference Workload
+
+Questions: jhylands@mozilla.com
+
+To generate these files:
+
+  Contacts:
+    - generate CSV file from http://www.fakenamegenerator.com/
+      - include: GivenName,MiddleInitial,Surname,StreetAddress,City,State,ZipCode,CountryFull,EmailAddress,TelephoneNumber,Birthday,Occupation,Company
+    - convert CSV to JSON
+    - move JSON file to: gaia/dev_apps/uitest/data/fakecontacts/fakecontacts.json
+      - works best if you use b2g-desktop
+    - if you're using b2g-desktop, cd gaia and run make
+    - if you're using a real phone, rebuild, flash
+    - run 'UI tests' app on phone/b2g-desktop
+      - 'Contact tests'
+        - 'Insert fake contacts'
+    - once its done, and contacts are there, from terminal on desktop (in gaia directory):
+      - abd pull /data/local/indexedDB/chrome/3406066227csotncta.sqlite test_media/reference-workload/contactsDb-XXX.sqlite
+        - XXX is # of contacts
+    - if you're using b2g-desktop:
+      - cp profile/indexedDB/chrome/3406066227csotncta.sqlite test_media/reference-workload/contactsDb-XXX.sqlite
+
+  SMS
+    - import patch associated with bug 806598 to your device repository
+    - ensure you have b2g-desktop repository set up, with XULRunner built with RIL enabled (see mozconfig)
+    - ensure your device is plugged in via usb and ready for adb
+    - flash the phone fresh
+    - ensure the sms database in gaia/profile/indexedDB/chrome is empty (should be ~524.3 kB)
+    - from terminal on desktop (in gaia directory):
+      - make fake-sms
+      - adb pull /data/local/indexedDB/chrome/226660312ssm.sqlite test_media/reference-workload/smsDb-XXX.sqlite
+        - XXX is # of messaged generated
+
+