Publish Vulpes Preview 15 sources for desktop, Android and Tundra
Share Gaia assets and adapters, keep private profiles and development documents local, and update both public READMEs.
This commit is contained in:
OverSu
parent
93e3a48004
commit
b38dea0597
25901 changed files with 9982925 additions and 369 deletions
|
|
@ -0,0 +1,18 @@
|
|||
Metadata-Version: 1.1
|
||||
Name: gaia-runner-service
|
||||
Version: 0.3
|
||||
Summary: A utility for running tests over a socket from JS to python
|
||||
Home-page: UNKNOWN
|
||||
Author: Andrew Halberstadt
|
||||
Author-email: ahalberstadt@mozilla.com
|
||||
License: MPL 2.0
|
||||
Description: See https://github.com/mozilla-b2g/gaia/tree/master/tests/python/runner-service
|
||||
Keywords: mozilla
|
||||
Platform: UNKNOWN
|
||||
Classifier: Environment :: Console
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
|
||||
Classifier: Natural Language :: English
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
setup.py
|
||||
gaia_runner_service.egg-info/PKG-INFO
|
||||
gaia_runner_service.egg-info/SOURCES.txt
|
||||
gaia_runner_service.egg-info/dependency_links.txt
|
||||
gaia_runner_service.egg-info/entry_points.txt
|
||||
gaia_runner_service.egg-info/not-zip-safe
|
||||
gaia_runner_service.egg-info/requires.txt
|
||||
gaia_runner_service.egg-info/top_level.txt
|
||||
runner_service/__init__.py
|
||||
runner_service/listener.py
|
||||
runner_service/runintegration.py
|
||||
runner_service/handlers/__init__.py
|
||||
runner_service/handlers/runner.py
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
|
||||
[console_scripts]
|
||||
gaia-integration=runner_service:runintegration.cli
|
||||
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
mozrunner >= 6.11
|
||||
mozprofile >= 0.21
|
||||
mozcrash >= 0.16
|
||||
|
|
@ -0,0 +1 @@
|
|||
runner_service
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import runintegration
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from . import runner
|
||||
|
||||
runner_handlers = {
|
||||
'desktop': runner.DesktopHandler,
|
||||
'device': runner.DeviceHandler,
|
||||
'emulator': runner.EmulatorHandler,
|
||||
}
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from abc import ABCMeta, abstractmethod, abstractproperty
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import sys
|
||||
|
||||
from mozprofile import Profile
|
||||
from mozrunner import (
|
||||
B2GDesktopRunner,
|
||||
B2GDeviceRunner,
|
||||
B2GEmulatorRunner
|
||||
)
|
||||
|
||||
# We log over a special pipe (fd3) which node knows to hook up.
|
||||
GECKO_LOG_FD = 3
|
||||
|
||||
class DeviceUtils:
|
||||
def __init__(self, deviceManager):
|
||||
self.dm = deviceManager
|
||||
|
||||
def _purge_data(self):
|
||||
# Clean up data on device.
|
||||
print("cleaning up data on device.")
|
||||
data_paths = ['/cache/*',
|
||||
'/data/local/debug_info_trigger',
|
||||
'/data/local/indexedDB',
|
||||
'/data/local/OfflineCache',
|
||||
'/data/local/permissions.sqlite',
|
||||
'/data/local/storage/persistent',
|
||||
'/data/local/storage/default']
|
||||
for path in data_paths:
|
||||
if self.dm.dirExists(path):
|
||||
self.dm.removeDir(path)
|
||||
|
||||
def _purge_wifi_networks(self):
|
||||
# Remove remembered networks
|
||||
print("removing remembered wifi networks.")
|
||||
self.dm.removeDir('/data/misc/wifi/wpa_supplicant.conf')
|
||||
|
||||
def _purge_storage_data(self):
|
||||
# Clean up storage.
|
||||
print("cleaning up storage on device.")
|
||||
# TODO: Remove hard-coded paths once bug 1018079 is resolved
|
||||
storage_paths = ['/mnt/sdcard/',
|
||||
'/mnt/extsdcard/',
|
||||
'/storage/sdcard/',
|
||||
'/storage/sdcard0/',
|
||||
'/storage/sdcard1/']
|
||||
for path in storage_paths:
|
||||
if self.dm.dirExists(path):
|
||||
for item in self.dm.listFiles(path):
|
||||
self.dm.removeDir('/'.join([path, item]))
|
||||
|
||||
def _purge_installed_apps(self):
|
||||
# Clean up apps on device.
|
||||
apps = json.loads(self.dm.pullFile('/data/local/webapps/webapps.json'))
|
||||
system_install_time = apps['system.gaiamobile.org']['installTime']
|
||||
for app in apps.values():
|
||||
if app.get('installTime') > system_install_time:
|
||||
# removing any webapps installed since build time
|
||||
path = posixpath.join(app.get('basePath'), app.get('id'))
|
||||
print('Removing %s' % path)
|
||||
self.dm.removeDir(path)
|
||||
|
||||
def _stop_b2g(self):
|
||||
print("stopping b2g on device before starting runner.")
|
||||
self.dm.shellCheckOutput(['stop', 'b2g'])
|
||||
|
||||
|
||||
class MozrunnerHandler(object):
|
||||
__metaclass__ = ABCMeta
|
||||
runner = None
|
||||
|
||||
def __init__(self, symbols_path=None, *args, **kwargs):
|
||||
fd = os.fdopen(GECKO_LOG_FD, 'a')
|
||||
process_args = {
|
||||
'stream': fd,
|
||||
'onFinish': self.on_finish
|
||||
}
|
||||
|
||||
dump_path = kwargs.pop('dump_path', os.path.join(os.getcwd(), "minidumps"))
|
||||
self.common_runner_args = {
|
||||
'process_args': process_args,
|
||||
'symbols_path': symbols_path,
|
||||
'dump_save_path': dump_path,
|
||||
|
||||
'env': {
|
||||
# Required for crash reporting...
|
||||
'MOZ_CRASHREPORTER': '1',
|
||||
'MOZ_CRASHREPORTER_NO_REPORT': '1'
|
||||
}
|
||||
}
|
||||
|
||||
# Ensure we set the DISPLAY if it is set in the current environment so
|
||||
# things like Xvfb work correctly...
|
||||
if 'DISPLAY' in os.environ:
|
||||
self.common_runner_args['env']['DISPLAY'] = os.environ['DISPLAY']
|
||||
|
||||
@abstractmethod
|
||||
def start_runner(self, target, options):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def stop_runner(self):
|
||||
pass
|
||||
|
||||
def on_finish(self):
|
||||
pass
|
||||
|
||||
def cleanup(self):
|
||||
if self.runner:
|
||||
self.runner.cleanup()
|
||||
|
||||
|
||||
class DesktopHandler(MozrunnerHandler):
|
||||
|
||||
def start_runner(self, binary, options):
|
||||
profile = Profile(profile=options.get('profile'))
|
||||
cmdargs = options.get('argv', [])
|
||||
|
||||
if 'screen' in options and \
|
||||
'width' in options['screen'] and \
|
||||
'height' in options['screen']:
|
||||
screen = '--screen=%sx%s' % (options['screen']['width'], \
|
||||
options['screen']['height'])
|
||||
|
||||
if 'dpi' in options['screen']:
|
||||
screen = '%s@%s' % (screen, options['screen']['dpi'])
|
||||
cmdargs.append(screen)
|
||||
|
||||
if options.get('noRemote', True):
|
||||
cmdargs.append('-no-remote')
|
||||
|
||||
if options.get('url'):
|
||||
cmdargs.append(options['url'])
|
||||
|
||||
if options.get('chrome'):
|
||||
cmdargs.extend(['-chrome', options['chrome']])
|
||||
|
||||
if options.get('startDebugger'):
|
||||
cmdargs.extend(['-start-debugger-server', options['startDebugger']])
|
||||
|
||||
self.runner = B2GDesktopRunner(binary, cmdargs=cmdargs, profile=profile,
|
||||
**self.common_runner_args)
|
||||
self.runner.start()
|
||||
|
||||
def stop_runner(self):
|
||||
self.runner.stop()
|
||||
|
||||
|
||||
class EmulatorHandler(MozrunnerHandler):
|
||||
|
||||
def __init__(self, b2g_home, *args, **kwargs):
|
||||
MozrunnerHandler.__init__(self, *args, **kwargs)
|
||||
|
||||
self.runner = B2GEmulatorRunner(b2g_home=b2g_home, **self.common_runner_args)
|
||||
self.runner.device.start()
|
||||
|
||||
def start_runner(self, binary, options):
|
||||
port = options.get('port')
|
||||
profile = Profile(profile=options.get('profile'))
|
||||
|
||||
self.runner.device.setup_port_forwarding(local_port=port, remote_port=port)
|
||||
|
||||
self.runner.profile = profile
|
||||
self.runner.start()
|
||||
if not self.runner.device.wait_for_port(port):
|
||||
raise Exception("Wait for port timed out")
|
||||
|
||||
def stop_runner(self):
|
||||
self.runner.stop()
|
||||
|
||||
|
||||
class DeviceHandler(MozrunnerHandler):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
serial = kwargs.pop('serial', None)
|
||||
MozrunnerHandler.__init__(self, *args, **kwargs)
|
||||
|
||||
self.runner = B2GDeviceRunner(serial=serial, **self.common_runner_args)
|
||||
self.runner.device.connect()
|
||||
|
||||
def start_runner(self, target, options):
|
||||
du = DeviceUtils(self.runner.device.dm)
|
||||
device = self.runner.device
|
||||
port = options.get('port')
|
||||
profile = Profile(profile=options.get('profile'))
|
||||
|
||||
device.setup_port_forwarding(local_port=port, remote_port=port)
|
||||
|
||||
du._stop_b2g()
|
||||
du._purge_data()
|
||||
du._purge_wifi_networks()
|
||||
du._purge_storage_data()
|
||||
|
||||
self.runner.profile = profile
|
||||
self.runner.start()
|
||||
|
||||
if not device.wait_for_port(port=port, timeout=600):
|
||||
raise Exception("Wait for port timed out")
|
||||
|
||||
def stop_runner(self):
|
||||
self.runner.stop()
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import socket
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import uuid
|
||||
import BaseHTTPServer
|
||||
import traceback
|
||||
|
||||
import mozcrash
|
||||
|
||||
from .handlers import runner_handlers
|
||||
|
||||
class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
REQUESTS = {}
|
||||
INACTIVE = []
|
||||
|
||||
def log_message(self, format, *args):
|
||||
'''
|
||||
Override log message to bypass certain behaviors which assume tcp
|
||||
sockets
|
||||
'''
|
||||
pass
|
||||
|
||||
def log_error(self, format, *args):
|
||||
'''
|
||||
Override log error to bypass certain behaviors which assume tcp
|
||||
sockets
|
||||
'''
|
||||
print(args)
|
||||
|
||||
def send_json(self, status, payload):
|
||||
'''
|
||||
Respond to incoming http request...
|
||||
|
||||
:param int status:
|
||||
:param dict payload: payload to send as json to the socket.
|
||||
'''
|
||||
payload = json.dumps(payload)
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Length', len(payload))
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def do_GET(self):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
content_length = self.headers.getheader('content-length', 0)
|
||||
payload = json.loads(self.rfile.read(int(content_length)))
|
||||
handler = self.path[1:]
|
||||
method = 'do_' + handler
|
||||
|
||||
# No matter what we send application/json
|
||||
if hasattr(self, method):
|
||||
try:
|
||||
func = getattr(self, method)
|
||||
result = func(payload)
|
||||
except Exception as e:
|
||||
self.send_json(500, {
|
||||
'message': str(e),
|
||||
'stack': traceback.format_exc()
|
||||
})
|
||||
else:
|
||||
self.send_json(200, result)
|
||||
else:
|
||||
self.send_json(404, { message: 'Unknown path :' + self.path })
|
||||
|
||||
|
||||
def do_test_end(self, data):
|
||||
print 'TEST-END | %s' % data['test']
|
||||
|
||||
def do_test_start(self, data):
|
||||
print 'TEST-START | %s' % data['test']
|
||||
|
||||
def do_test_status(self, data):
|
||||
print 'TEST-STATUS | %s | %s' % (data['subtest'], data['status'])
|
||||
|
||||
def do_connect(self, payload):
|
||||
'''
|
||||
Simple connection ready hook to signal socket readiness...
|
||||
'''
|
||||
return {}
|
||||
|
||||
def do_get_crash_info(self, payload):
|
||||
# Somewhat terrible (maybe) hack to get CrashInfo private interface.
|
||||
crash_info = mozcrash.mozcrash.CrashInfo(**payload)
|
||||
|
||||
# Return first error or nothing...
|
||||
for info in crash_info:
|
||||
return {
|
||||
'signature': info.signature,
|
||||
'stackwalk_stdout': info.stackwalk_stdout,
|
||||
'stackwalk_stderr': info.stackwalk_stderr,
|
||||
'stackwalk_retcode': info.stackwalk_retcode,
|
||||
'stackwalk_errors': info.stackwalk_errors
|
||||
}
|
||||
|
||||
return {}
|
||||
|
||||
def do_start_runner(self, payload):
|
||||
'''
|
||||
Begin a runner
|
||||
|
||||
:param str|None binary: Target binary (usually b2g-bin) for desktop.
|
||||
:param dict options: Options specific to the buildapp type.
|
||||
'''
|
||||
binary = payload.get('binary')
|
||||
options = payload.get('options', {})
|
||||
|
||||
handler_args = {
|
||||
'symbols_path': options.get('symbols_path')
|
||||
}
|
||||
|
||||
if 'b2g_home' in options:
|
||||
handler_args['b2g_home'] = options['b2g_home']
|
||||
|
||||
if options['buildapp'] == 'device' and 'serial' in options:
|
||||
handler_args['serial'] = options['serial']
|
||||
|
||||
if 'dump_path' in options:
|
||||
handler_args['dump_path'] = options['dump_path']
|
||||
|
||||
start_id = str(uuid.uuid4())
|
||||
handler = runner_handlers[options['buildapp']](**handler_args)
|
||||
handler.start_runner(binary, options)
|
||||
|
||||
print "Started runner with id = {0}".format(start_id)
|
||||
self.REQUESTS[start_id] = handler
|
||||
|
||||
return { 'id': start_id }
|
||||
|
||||
def do_stop_runner(self, payload):
|
||||
print "Stopping runner with id = {0}".format(payload['id'])
|
||||
payload_id = payload['id']
|
||||
if not payload_id in self.REQUESTS:
|
||||
if payload_id in self.INACTIVE:
|
||||
sys.stderr.write(
|
||||
'Received request to stop ' +
|
||||
payload_id +
|
||||
' but already stopped so nothing to do.\n')
|
||||
return {}
|
||||
runner_ids = str.join(
|
||||
', ',
|
||||
map(lambda x: str(x), self.REQUESTS.keys() + self.INACTIVE))
|
||||
msg = 'Cannot stop %s - service has %s' % (payload_id, runner_ids)
|
||||
raise Exception(msg)
|
||||
|
||||
self.INACTIVE.append(payload_id)
|
||||
handler = self.REQUESTS.pop(payload_id)
|
||||
handler.stop_runner()
|
||||
handler.cleanup()
|
||||
return {}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from argparse import ArgumentParser
|
||||
import threading
|
||||
import sys
|
||||
import socket
|
||||
import os
|
||||
import time
|
||||
|
||||
import SocketServer
|
||||
|
||||
from .listener import RequestHandler
|
||||
|
||||
class Server(SocketServer.ThreadingMixIn, SocketServer.UnixStreamServer):
|
||||
pass
|
||||
|
||||
def cli(args=sys.argv[1:]):
|
||||
'''
|
||||
Begin the server note that the majority of arguments will be passed via the
|
||||
cli but over the unix socket instead...
|
||||
'''
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('--path',
|
||||
dest='path',
|
||||
default='/tmp/marionette_socket_host_worker',
|
||||
help='Path to bind unix socket to'
|
||||
)
|
||||
args = parser.parse_args(args)
|
||||
|
||||
# Make sure the socket does not already exist.
|
||||
try:
|
||||
# XXX: This is potentially a bad thing maybe we should raise or warn
|
||||
# here...
|
||||
os.unlink(args.path)
|
||||
except OSError:
|
||||
if os.path.exists(args.path):
|
||||
raise
|
||||
|
||||
server = Server(args.path, RequestHandler)
|
||||
server.timeout = 300
|
||||
server_thread = threading.Thread(target=server.serve_forever)
|
||||
|
||||
server_thread.deamon = True
|
||||
server.running = True
|
||||
server_thread.start()
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(cli())
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
# NOTE: import the following due to linux bug http://bugs.python.org/issue15881
|
||||
try:
|
||||
import multiprocessing
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
PACKAGE_VERSION = '0.4'
|
||||
|
||||
deps = ['mozrunner >= 6.11',
|
||||
'mozprofile >= 0.21']
|
||||
|
||||
setup(name='gaia-runner-service',
|
||||
version=PACKAGE_VERSION,
|
||||
description='A utility for running tests over a socket from JS to python',
|
||||
long_description='See https://github.com/mozilla-b2g/gaia/tree/master/tests/python/runner-service',
|
||||
classifiers=['Environment :: Console',
|
||||
'Intended Audience :: Developers',
|
||||
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
|
||||
'Natural Language :: English',
|
||||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
keywords='mozilla',
|
||||
author='Andrew Halberstadt',
|
||||
author_email='ahalberstadt@mozilla.com',
|
||||
license='MPL 2.0',
|
||||
packages=find_packages(),
|
||||
include_package_data=True,
|
||||
zip_safe=False,
|
||||
install_requires=deps,
|
||||
entry_points="""
|
||||
[console_scripts]
|
||||
gaia-integration=runner_service:runintegration.cli
|
||||
""")
|
||||
Loading…
Reference in a new issue