summaryrefslogtreecommitdiffstats
path: root/taskcluster/taskgraph
diff options
context:
space:
mode:
Diffstat (limited to 'taskcluster/taskgraph')
-rw-r--r--taskcluster/taskgraph/actions/add_new_jobs.py1
-rw-r--r--taskcluster/taskgraph/actions/add_talos.py1
-rw-r--r--taskcluster/taskgraph/actions/backfill.py1
-rw-r--r--taskcluster/taskgraph/actions/cancel.py1
-rw-r--r--taskcluster/taskgraph/actions/cancel_all.py1
-rw-r--r--taskcluster/taskgraph/actions/create_interactive.py1
-rw-r--r--taskcluster/taskgraph/actions/purge_caches.py1
-rw-r--r--taskcluster/taskgraph/actions/registry.py3
-rw-r--r--taskcluster/taskgraph/actions/release_promotion.py1
-rw-r--r--taskcluster/taskgraph/actions/rerun.py61
-rw-r--r--taskcluster/taskgraph/actions/retrigger.py176
-rw-r--r--taskcluster/taskgraph/actions/retrigger_mochitest.py1
-rw-r--r--taskcluster/taskgraph/actions/run_missing_tests.py1
-rw-r--r--taskcluster/taskgraph/transforms/release_mark_as_started.py57
-rw-r--r--taskcluster/taskgraph/transforms/task.py26
-rw-r--r--taskcluster/taskgraph/util/scriptworker.py1
16 files changed, 179 insertions, 155 deletions
diff --git a/taskcluster/taskgraph/actions/add_new_jobs.py b/taskcluster/taskgraph/actions/add_new_jobs.py
index 6f1f19b..7353006 100644
--- a/taskcluster/taskgraph/actions/add_new_jobs.py
+++ b/taskcluster/taskgraph/actions/add_new_jobs.py
@@ -18,7 +18,6 @@ from .util import (
@register_callback_action(
name='add-new-jobs',
title='Add new jobs',
- kind='hook',
generic=True,
symbol='add-new',
description="Add new jobs using task labels.",
diff --git a/taskcluster/taskgraph/actions/add_talos.py b/taskcluster/taskgraph/actions/add_talos.py
index cae1a52..8941dd7 100644
--- a/taskcluster/taskgraph/actions/add_talos.py
+++ b/taskcluster/taskgraph/actions/add_talos.py
@@ -17,7 +17,6 @@ logger = logging.getLogger(__name__)
@register_callback_action(
name='run-all-talos',
title='Run All Talos Tests',
- kind='hook',
generic=True,
symbol='raT',
description="Add all Talos tasks to a push.",
diff --git a/taskcluster/taskgraph/actions/backfill.py b/taskcluster/taskgraph/actions/backfill.py
index 7650ae4..df62ca5 100644
--- a/taskcluster/taskgraph/actions/backfill.py
+++ b/taskcluster/taskgraph/actions/backfill.py
@@ -25,7 +25,6 @@ logger = logging.getLogger(__name__)
@register_callback_action(
title='Backfill',
name='backfill',
- kind='hook',
generic=True,
symbol='Bk',
description=('Take the label of the current task, '
diff --git a/taskcluster/taskgraph/actions/cancel.py b/taskcluster/taskgraph/actions/cancel.py
index da8cf0b..1d26756 100644
--- a/taskcluster/taskgraph/actions/cancel.py
+++ b/taskcluster/taskgraph/actions/cancel.py
@@ -14,7 +14,6 @@ from .registry import register_callback_action
title='Cancel Task',
name='cancel',
symbol='cx',
- kind='hook',
generic=True,
description=(
'Cancel the given task'
diff --git a/taskcluster/taskgraph/actions/cancel_all.py b/taskcluster/taskgraph/actions/cancel_all.py
index 56ae31b..f4ea1e6 100644
--- a/taskcluster/taskgraph/actions/cancel_all.py
+++ b/taskcluster/taskgraph/actions/cancel_all.py
@@ -23,7 +23,6 @@ logger = logging.getLogger(__name__)
@register_callback_action(
title='Cancel All',
name='cancel-all',
- kind='hook',
generic=True,
symbol='cAll',
description=(
diff --git a/taskcluster/taskgraph/actions/create_interactive.py b/taskcluster/taskgraph/actions/create_interactive.py
index f4496e3..cf2ae06 100644
--- a/taskcluster/taskgraph/actions/create_interactive.py
+++ b/taskcluster/taskgraph/actions/create_interactive.py
@@ -76,7 +76,6 @@ def context(params):
title='Create Interactive Task',
name='create-interactive',
symbol='create-inter',
- kind='hook',
generic=True,
description=(
'Create a a copy of the task that you can interact with'
diff --git a/taskcluster/taskgraph/actions/purge_caches.py b/taskcluster/taskgraph/actions/purge_caches.py
index 03ccc15..8b56a1f 100644
--- a/taskcluster/taskgraph/actions/purge_caches.py
+++ b/taskcluster/taskgraph/actions/purge_caches.py
@@ -18,7 +18,6 @@ logger = logging.getLogger(__name__)
title='Purge Worker Caches',
name='purge-cache',
symbol='purge-cache',
- kind='hook',
generic=True,
description=(
'Purge any caches associated with this task '
diff --git a/taskcluster/taskgraph/actions/registry.py b/taskcluster/taskgraph/actions/registry.py
index 2fc848b..7931700 100644
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -52,7 +52,7 @@ def hash_taskcluster_yml(filename):
def register_callback_action(name, title, symbol, description, order=10000,
context=[], available=lambda parameters: True,
- schema=None, kind='task', generic=True, cb_name=None):
+ schema=None, kind='hook', generic=True, cb_name=None):
"""
Register an action callback that can be triggered from supporting
user interfaces, such as Treeherder.
@@ -275,6 +275,7 @@ def register_callback_action(name, title, symbol, description, order=10000,
mem['registered'] = True
callbacks[cb_name] = cb
+ return cb
return register_callback
diff --git a/taskcluster/taskgraph/actions/release_promotion.py b/taskcluster/taskgraph/actions/release_promotion.py
index 806f76f..c86262f 100644
--- a/taskcluster/taskgraph/actions/release_promotion.py
+++ b/taskcluster/taskgraph/actions/release_promotion.py
@@ -79,6 +79,7 @@ def get_flavors(graph_config, param):
title='Release Promotion',
symbol='${input.release_promotion_flavor}',
description="Promote a release.",
+ generic=False,
order=500,
context=[],
available=is_release_promotion_available,
diff --git a/taskcluster/taskgraph/actions/rerun.py b/taskcluster/taskgraph/actions/rerun.py
deleted file mode 100644
index 9e314cd..0000000
--- a/taskcluster/taskgraph/actions/rerun.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import logging
-import sys
-
-from taskgraph.util.taskcluster import (
- status_task,
- rerun_task
-)
-from .registry import register_callback_action
-from .util import fetch_graph_and_labels
-
-logger = logging.getLogger(__name__)
-
-RERUN_STATES = ('exception', 'failed')
-
-
-@register_callback_action(
- title='Rerun',
- name='rerun',
- kind='hook',
- generic=True,
- symbol='rr',
- description=(
- 'Rerun a task.\n\n'
- 'This only works on failed or exception tasks in the original taskgraph,'
- ' and is CoT friendly.'
- ),
- order=300,
- context=[{}],
- schema={
- 'type': 'object',
- 'properties': {}
- }
-)
-def rerun_action(parameters, graph_config, input, task_group_id, task_id, task):
- parameters = dict(parameters)
- decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
- parameters, graph_config)
- label = task['metadata']['name']
- if task_id not in label_to_taskid.values():
- logger.error(
- "Refusing to rerun {}: taskId {} not in decision task {} label_to_taskid!".format(
- label, task_id, decision_task_id
- )
- )
-
- status = status_task(task_id)
- if status not in RERUN_STATES:
- logger.error(
- "Refusing to rerun {}: state {} not in {}!".format(label, status, RERUN_STATES)
- )
- sys.exit(1)
- rerun_task(task_id)
- logger.info('Reran {}'.format(label))
diff --git a/taskcluster/taskgraph/actions/retrigger.py b/taskcluster/taskgraph/actions/retrigger.py
index 8580231..402e59a 100644
--- a/taskcluster/taskgraph/actions/retrigger.py
+++ b/taskcluster/taskgraph/actions/retrigger.py
@@ -6,6 +6,8 @@
from __future__ import absolute_import, print_function, unicode_literals
+import sys
+
import logging
import textwrap
@@ -18,15 +20,17 @@ from .util import (
create_task_from_def,
)
from .registry import register_callback_action
+from ..util import taskcluster
logger = logging.getLogger(__name__)
+RERUN_STATES = ('exception', 'failed')
+
@register_callback_action(
title='Retrigger',
name='retrigger',
symbol='rt',
- kind='hook',
cb_name='retrigger-decision',
description=textwrap.dedent('''\
Create a clone of the task (retriggering decision, action, and cron tasks requires
@@ -53,13 +57,12 @@ def retrigger_decision_action(parameters, graph_config, input, task_group_id, ta
title='Retrigger',
name='retrigger',
symbol='rt',
- kind='hook',
generic=True,
description=(
'Create a clone of the task.'
),
order=19, # must be greater than other orders in this file, as this is the fallback version
- context=[{}],
+ context=[{'retrigger': 'true'}],
schema={
'type': 'object',
'properties': {
@@ -82,6 +85,48 @@ def retrigger_decision_action(parameters, graph_config, input, task_group_id, ta
}
}
)
+@register_callback_action(
+ title='Retrigger (disabled)',
+ name='retrigger',
+ cb_name='retrigger-disabled',
+ symbol='rt',
+ generic=True,
+ description=(
+ 'Create a clone of the task.\n\n'
+ 'This type of task should typically be re-run instead of re-triggered.'
+ ),
+ order=20, # must be greater than other orders in this file, as this is the fallback version
+ context=[{}],
+ schema={
+ 'type': 'object',
+ 'properties': {
+ 'downstream': {
+ 'type': 'boolean',
+ 'description': (
+ 'If true, downstream tasks from this one will be cloned as well. '
+ 'The dependencies will be updated to work with the new task at the root.'
+ ),
+ 'default': False,
+ },
+ 'times': {
+ 'type': 'integer',
+ 'default': 1,
+ 'minimum': 1,
+ 'maximum': 100,
+ 'title': 'Times',
+ 'description': 'How many times to run each task.',
+ },
+ 'force': {
+ 'type': 'boolean',
+ 'default': False,
+ 'description': (
+ 'This task should not be re-triggered. '
+ 'This can be overridden by passing `true` here.'
+ ),
+ },
+ }
+ }
+)
def retrigger_action(parameters, graph_config, input, task_group_id, task_id, task):
decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
parameters, graph_config)
@@ -91,6 +136,15 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id, ta
with_downstream = ' '
to_run = [label]
+ if not input.get('force', None) and not full_task_graph[label].attributes.get('retrigger'):
+ logger.info(
+ "Not retriggering task {}, task should not be retrigged "
+ "and force not specified.".format(
+ label
+ )
+ )
+ sys.exit(1)
+
if input.get('downstream'):
to_run = full_task_graph.graph.transitive_closure(set(to_run), reverse=True).nodes
to_run = to_run & set(label_to_taskid.keys())
@@ -102,3 +156,119 @@ def retrigger_action(parameters, graph_config, input, task_group_id, task_id, ta
logger.info('Scheduled {}{}(time {}/{})'.format(label, with_downstream, i+1, times))
combine_task_graph_files(list(range(times)))
+
+
+@register_callback_action(
+ title='Rerun',
+ name='rerun',
+ generic=True,
+ symbol='rr',
+ description=(
+ 'Rerun a task.\n\n'
+ 'This only works on failed or exception tasks in the original taskgraph,'
+ ' and is CoT friendly.'
+ ),
+ order=300,
+ context=[{}],
+ schema={
+ 'type': 'object',
+ 'properties': {}
+ }
+)
+def rerun_action(parameters, graph_config, input, task_group_id, task_id, task):
+ parameters = dict(parameters)
+ decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
+ parameters, graph_config)
+ label = task['metadata']['name']
+ if task_id not in label_to_taskid.values():
+ logger.error(
+ "Refusing to rerun {}: taskId {} not in decision task {} label_to_taskid!".format(
+ label, task_id, decision_task_id
+ )
+ )
+
+ _rerun_task(task_id, label)
+
+
+def _rerun_task(task_id, label):
+ status = taskcluster.status_task(task_id)
+ if status not in RERUN_STATES:
+ logger.warning(
+ "No need to to rerun {}: state '{}' not in {}!".format(label, status, RERUN_STATES)
+ )
+ return
+ taskcluster.rerun_task(task_id)
+ logger.info('Reran {}'.format(label))
+
+
+@register_callback_action(
+ title='Retrigger',
+ name='retrigger-multiple',
+ symbol='rt',
+ generic=True,
+ description=(
+ 'Create a clone of the task.'
+ ),
+ context=[],
+ schema={
+ "type": "object",
+ "properties": {
+ "requests": {
+ "type": "array",
+ "items": {
+ "tasks": {
+ "type": "array",
+ 'description': 'An array of task labels',
+ 'items': {
+ 'type': 'string'
+ }
+ },
+ "times": {
+ "type": "integer",
+ "minimum": 1,
+ "maximum": 100,
+ "title": "Times",
+ "description": "How many times to run each task.",
+ },
+ "additionalProperties": False,
+ },
+ },
+ "additionalProperties": False,
+ },
+ },
+)
+def retrigger_multiple(parameters, graph_config, input, task_group_id, task_id, task):
+ decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
+ parameters, graph_config)
+
+ suffixes = []
+ for i, request in enumerate(input.get('requests', [])):
+ times = request.get('times', 1)
+ rerun_tasks = [
+ label for label in request.get('tasks')
+ if not full_task_graph[label].attributes.get('retrigger')]
+ retrigger_tasks = [
+ label for label in request.get('tasks')
+ if full_task_graph[label].attributes.get('retrigger')
+ ]
+
+ for label in rerun_tasks:
+ # XXX we should not re-run tasks pulled in from other pushes
+ # In practice, this shouldn't matter, as only completed tasks
+ # are pulled in from other pushes and treeherder won't pass
+ # those labels.
+ _rerun_task(label_to_taskid[label], label)
+
+ for j in xrange(times):
+ suffix = '{}-{}'.format(i, j)
+ suffixes.append(suffix)
+ create_tasks(
+ retrigger_tasks,
+ full_task_graph,
+ label_to_taskid,
+ parameters,
+ decision_task_id,
+ suffix,
+ )
+
+ combine_task_graph_files(suffixes)
diff --git a/taskcluster/taskgraph/actions/retrigger_mochitest.py b/taskcluster/taskgraph/actions/retrigger_mochitest.py
index 467e068..c5cffa7 100644
--- a/taskcluster/taskgraph/actions/retrigger_mochitest.py
+++ b/taskcluster/taskgraph/actions/retrigger_mochitest.py
@@ -24,7 +24,6 @@ logger = logging.getLogger(__name__)
name='retrigger-mochitest',
title='Retrigger Mochitest/Reftest with Debugging',
symbol='rt',
- kind='hook',
generic=True,
description="Retriggers the specified mochitest/reftest job with additional options",
context=[{'test-type': 'mochitest'},
diff --git a/taskcluster/taskgraph/actions/run_missing_tests.py b/taskcluster/taskgraph/actions/run_missing_tests.py
index 9b4f028..ab28c49 100644
--- a/taskcluster/taskgraph/actions/run_missing_tests.py
+++ b/taskcluster/taskgraph/actions/run_missing_tests.py
@@ -18,7 +18,6 @@ logger = logging.getLogger(__name__)
@register_callback_action(
name='run-missing-tests',
title='Run Missing Tests',
- kind='hook',
generic=True,
symbol='rmt',
description=(
diff --git a/taskcluster/taskgraph/transforms/release_mark_as_started.py b/taskcluster/taskgraph/transforms/release_mark_as_started.py
deleted file mode 100644
index a39ae6a..0000000
--- a/taskcluster/taskgraph/transforms/release_mark_as_started.py
+++ /dev/null
@@ -1,57 +0,0 @@
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Add from parameters.yml into Balrog publishing tasks.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import json
-
-from taskgraph.transforms.base import TransformSequence
-from taskgraph.transforms.l10n import parse_locales_file
-from taskgraph.util.schema import resolve_keyed_by
-from taskgraph.util.scriptworker import get_release_config
-
-transforms = TransformSequence()
-
-
-@transforms.add
-def make_task_description(config, jobs):
- release_config = get_release_config(config)
- for job in jobs:
- resolve_keyed_by(
- job, 'worker-type', item_name=job['name'], project=config.params['project']
- )
- resolve_keyed_by(
- job, 'scopes', item_name=job['name'], project=config.params['project']
- )
-
- job['worker']['release-name'] = '{product}-{version}-build{build_number}'.format(
- product=job['shipping-product'].capitalize(),
- version=release_config['version'],
- build_number=release_config['build_number']
- )
- job['worker']['product'] = job['shipping-product']
- branch = config.params['head_repository'].split('https://hg.mozilla.org/')[1]
- job['worker']['branch'] = branch
-
- # locales files has different structure between mobile and desktop
- locales_file = job['locales-file']
- all_locales = {}
-
- if job['shipping-product'] == 'fennec':
- with open(locales_file, mode='r') as f:
- all_locales = json.dumps(json.load(f))
- else:
- all_locales = "\n".join([
- "{} {}".format(locale, revision)
- for locale, revision in sorted(parse_locales_file(job['locales-file']).items())
- ])
-
- job['worker']['locales'] = all_locales
- del job['locales-file']
-
- yield job
diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py
index 9a82d07..fa44118 100644
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1140,28 +1140,6 @@ def build_ship_it_shipped_payload(config, task, task_def):
}
-@payload_builder('shipit-started', schema={
- Required('release-name'): basestring,
- Required('product'): basestring,
- Required('branch'): basestring,
- Required('locales'): basestring,
-})
-def build_ship_it_started_payload(config, task, task_def):
- worker = task['worker']
- release_config = get_release_config(config)
-
- task_def['payload'] = {
- 'release_name': worker['release-name'],
- 'product': worker['product'],
- 'version': release_config['version'],
- 'build_number': release_config['build_number'],
- 'branch': worker['branch'],
- 'revision': get_branch_rev(config),
- 'partials': release_config.get('partial_versions', ""),
- 'l10n_changesets': worker['locales'],
- }
-
-
@payload_builder('sign-and-push-addons', schema={
Required('channel'): Any('listed', 'unlisted'),
Required('upstream-artifacts'): [{
@@ -1654,10 +1632,13 @@ def build_task(config, tasks):
DEFAULT_BRANCH_PRIORITY)
tags = task.get('tags', {})
+ attributes = task.get('attributes', {})
+
tags.update({
'createdForUser': config.params['owner'],
'kind': config.kind,
'label': task['label'],
+ 'retrigger': 'true' if attributes.get('retrigger', False) else 'false'
})
task_def = {
@@ -1692,7 +1673,6 @@ def build_task(config, tasks):
# add the payload and adjust anything else as required (e.g., scopes)
payload_builders[task['worker']['implementation']](config, task, task_def)
- attributes = task.get('attributes', {})
# Resolve run-on-projects
build_platform = attributes.get('build_platform')
resolve_keyed_by(task, 'run-on-projects', item_name=task['label'],
diff --git a/taskcluster/taskgraph/util/scriptworker.py b/taskcluster/taskgraph/util/scriptworker.py
index 2ccc518..de79a01 100644
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -338,7 +338,6 @@ def get_release_config(config):
'release-secondary-update-verify-config',
'release-balrog-submit-toplevel',
'release-secondary-balrog-submit-toplevel',
- 'release-mark-as-started'
):
partial_updates = json.loads(partial_updates)
release_config['partial_versions'] = ', '.join([