diff --git a/.travis.yml b/.travis.yml index ef577c4..8dd2796 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,12 @@ language: python -python: -- '2.7' +matrix: + include: + - python: 2.7 + - python: 3.5 + +env: + BOTO_CONFIG=/dev/null + install: - make develop sudo: false diff --git a/Makefile b/Makefile index 297a2c9..b24dbcc 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,8 @@ .PHONY: develop: - # Hard to bootstrap these setup-requirements from setup.py unless we - # are happy to use easy_install. Lets pip them- - pip install pytest-runner setupext-pip~=1.0.5 - python setup.py requirements --install-test-requirements --install-extra-requirements documentation + pip install pipenv + pipenv install --dev --deploy python setup.py develop test: diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..6f6a75b --- /dev/null +++ b/Pipfile @@ -0,0 +1,9 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +celery-cloudwatch = {editable = true, path = "."} + +[dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..a95e1e7 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,101 @@ +{ + "_meta": { + "hash": { + "sha256": "3e01f7b7c0e014ab9451962fdf9050d60c0a13d62cc2766078dcd8f961959811" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "amqp": { + "hashes": [ + "sha256:073dd02fdd73041bffc913b767866015147b61f2a9bc104daef172fc1a0066eb", + "sha256:eed41946890cd43e8dee44a316b85cf6fee5a1a34bb4a562b660a358eb529e1b" + ], + "version": "==2.3.2" + }, + "billiard": { + "hashes": [ + "sha256:ed65448da5877b5558f19d2f7f11f8355ea76b3e63e1c0a6059f47cfae5f1c84" + ], + "version": "==3.5.0.4" + }, + "boto": { + "hashes": [ + "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8", + "sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a" + ], + "version": "==2.49.0" + }, + "celery": { + "hashes": [ + "sha256:77dab4677e24dc654d42dfbdfed65fa760455b6bb563a0877ecc35f4cfcfc678", + "sha256:ad7a7411772b80a4d6c64f2f7f723200e39fb66cf614a7fdfab76d345acc7b13" + ], + "version": "==4.2.1" + }, + "celery-cloudwatch": { + "editable": true, + "path": "." + }, + "kombu": { + "hashes": [ + "sha256:86adec6c60f63124e2082ea8481bbe4ebe04fde8ebed32c177c7f0cd2c1c9082", + "sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af" + ], + "version": "==4.2.1" + }, + "pytz": { + "hashes": [ + "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", + "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" + ], + "version": "==2018.5" + }, + "pyyaml": { + "hashes": [ + "sha256:3d7da3009c0f3e783b2c873687652d83b1bbfd5c88e9813fb7e5b03c0dd3108b", + "sha256:3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf", + "sha256:40c71b8e076d0550b2e6380bada1f1cd1017b882f7e16f09a65be98e017f211a", + "sha256:558dd60b890ba8fd982e05941927a3911dc409a63dcb8b634feaa0cda69330d3", + "sha256:a7c28b45d9f99102fa092bb213aa12e0aaf9a6a1f5e395d36166639c1f96c3a1", + "sha256:aa7dd4a6a427aed7df6fb7f08a580d68d9b118d90310374716ae90b710280af1", + "sha256:bc558586e6045763782014934bfaf39d48b8ae85a2713117d16c39864085c613", + "sha256:d46d7982b62e0729ad0175a9bc7e10a566fc07b224d2c79fafb5e032727eaa04", + "sha256:d5eef459e30b09f5a098b9cea68bebfeb268697f78d647bd255a085371ac7f3f", + "sha256:e01d3203230e1786cd91ccfdc8f8454c8069c91bee3962ad93b87a4b2860f537", + "sha256:e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531" + ], + "version": "==3.13" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + }, + "vine": { + "hashes": [ + "sha256:52116d59bc45392af9fdd3b75ed98ae48a93e822cee21e5fda249105c59a7a72", + "sha256:6849544be74ec3638e84d90bc1cf2e1e9224cc10d96cd4383ec3f69e9bce077b" + ], + "version": "==1.1.4" + }, + "voluptuous": { + "hashes": [ + "sha256:303542b3fc07fb52ec3d7a1c614b329cdbee13a9d681935353d8ea56a7bfa9f1", + "sha256:567a56286ef82a9d7ae0628c5842f65f516abcb496e74f3f59f1d7b28df314ef" + ], + "version": "==0.11.5" + } + }, + "develop": {} +} diff --git a/celery_cloudwatch/__main__.py b/celery_cloudwatch/__main__.py index a4026f5..b483ce9 100644 --- a/celery_cloudwatch/__main__.py +++ b/celery_cloudwatch/__main__.py @@ -2,7 +2,6 @@ import os import voluptuous as v import yaml -import six import logging import logging.config @@ -11,8 +10,8 @@ config_schema = v.Schema({ v.Optional('ccwatch', default={}): v.Schema({ - v.Optional('broker', default=None): v.Any(None, six.binary_type), - v.Optional('camera', default="celery_cloudwatch.CloudWatchCamera"): v.Any(str, six.binary_type), + v.Optional('broker', default=None): v.Any(None, str), + v.Optional('camera', default="celery_cloudwatch.CloudWatchCamera"): str, v.Optional('verbose', default=False): bool }, extra=False), v.Optional('camera', default={}): v.Schema({ @@ -21,20 +20,20 @@ }, extra=False), v.Optional('cloudwatch-camera', default={}): v.Schema({ v.Optional('dryrun', default=False): bool, - v.Optional('namespace', default='celery'): six.binary_type, + v.Optional('namespace', default='celery'): str, v.Optional('tasks', default=[]): v.Schema([ - six.binary_type, v.Schema({ - 'name': six.binary_type, + str, v.Schema({ + 'name': str, 'dimensions': v.Schema({ - v.Extra: six.binary_type + v.Extra: str }, extra=True) }, extra=False) ]), v.Optional('task-groups', default=[]): [ v.Schema({ - 'tasks': [six.binary_type], + 'tasks': [str], 'dimensions': v.Schema({ - v.Extra: six.binary_type + v.Extra: str }) }) ], diff --git a/celery_cloudwatch/cloudwatch_camera.py b/celery_cloudwatch/cloudwatch_camera.py index 165a6f0..cc91946 100644 --- a/celery_cloudwatch/cloudwatch_camera.py +++ b/celery_cloudwatch/cloudwatch_camera.py @@ -1,5 +1,6 @@ import json import logging +import six import sys import traceback @@ -40,14 +41,14 @@ def on_shutter(self, state): try: self.metrics = self._build_metrics(state) except RuntimeError as r: - print r + print(r) def after_shutter(self): try: self.metrics.send() except: - print "Exception in user code:" - print '-'*60 + print("Exception in user code:") + print('-'*60) traceback.print_exc(file=sys.stdout) finally: self.metrics = None @@ -86,7 +87,7 @@ def _build_metrics(self, state): def _add_task_events(self, metrics, task_event_sent, task_event_started, task_event_succeeded, task_event_failed, num_waiting_by_task, num_running_by_task, time_to_start, time_to_process): - for task_name, dimensions in self.task_mapping.iteritems(): + for task_name, dimensions in six.iteritems(self.task_mapping): metrics.add('CeleryEventSent', unit='Count', value=task_event_sent.get(task_name, 0), dimensions=dimensions) metrics.add('CeleryEventStarted', unit='Count', value=task_event_started.get(task_name, 0), dimensions=dimensions) metrics.add('CeleryEventSucceeded', unit='Count', value=task_event_succeeded.get(task_name, 0), dimensions=dimensions) @@ -143,7 +144,7 @@ def _add_task_groups(self, metrics, task_event_sent, task_event_started, task_ev def xchunk(arr, size): - for x in xrange(0, len(arr), size): + for x in six.moves.range(0, len(arr), size): yield arr[x:x+size] @@ -169,7 +170,7 @@ def _serialize(self, metric_chunk): } index = 0 for metric in metric_chunk: - for key, val in metric.serialize().iteritems(): + for key, val in six.iteritems(metric.serialize()): params['MetricData.member.%d.%s' % (index + 1, key)] = val index += 1 return params @@ -178,8 +179,8 @@ def send(self): for metric_chunk in xchunk(self.metrics, self._metric_chunk_size): metrics = self._serialize(metric_chunk) if self.verbose: - print 'PutMetricData' - print json.dumps(metrics, indent=2, sort_keys=True) + print('PutMetricData') + print(json.dumps(metrics, indent=2, sort_keys=True)) if self.aws_connection: self.aws_connection.get_status('PutMetricData', metrics, verb="POST") @@ -233,7 +234,7 @@ def _build_dimension_param(dimensions, params): for dim_name in dimensions: dim_value = dimensions[dim_name] if dim_value: - if isinstance(dim_value, basestring): + if isinstance(dim_value, six.string_types): dim_value = [dim_value] for value in dim_value: params['%s.%d.Name' % (prefix, i+1)] = dim_name diff --git a/celery_cloudwatch/print_camera.py b/celery_cloudwatch/print_camera.py index 031bb55..120ae09 100644 --- a/celery_cloudwatch/print_camera.py +++ b/celery_cloudwatch/print_camera.py @@ -8,42 +8,42 @@ class PrintCamera(Camera): clear_after = True def on_shutter(self, state): - print '-----' + print('-----') - print 'Time to Start' + print('Time to Start') total = Stats() for method_name, stats in state.time_to_start.items(): - print "%s avg:%.2fs, max:%.2fs, min: %.2fs" % (method_name, stats.average() or -1.0 , stats.maximum or -1.0, stats.minimum or -1.0) + print("%s avg:%.2fs, max:%.2fs, min: %.2fs" % (method_name, stats.average() or -1.0 , stats.maximum or -1.0, stats.minimum or -1.0)) total += stats - print '' - print 'Time to Process' + print('') + print('Time to Process') total = Stats() for method_name, stats in state.time_to_process.items(): - print "%s avg:%.2fs, max:%.2fs, min: %.2fs" % (method_name, stats.average() or -1.0, stats.maximum or -1.0, stats.minimum or -1.0) + print("%s avg:%.2fs, max:%.2fs, min: %.2fs" % (method_name, stats.average() or -1.0, stats.maximum or -1.0, stats.minimum or -1.0)) total += stats - print "Total: avg:%.2fs, max:%.2fs, min: %.2fs" % (total.average() or -1.0, total.maximum or -1.0, total.minimum or -1.0) + print("Total: avg:%.2fs, max:%.2fs, min: %.2fs" % (total.average() or -1.0, total.maximum or -1.0, total.minimum or -1.0)) - print '' - print 'Event Totals' + print('') + print('Event Totals') methods = set(state.task_event_sent.keys() + state.task_event_started.keys() + state.task_event_succeeded.keys() + state.task_event_failed.keys()) for method_name in methods: if method_name in state.task_event_sent: - print "%s[%s]: %d" % (method_name, 'waiting', state.task_event_sent[method_name]) + print("%s[%s]: %d" % (method_name, 'waiting', state.task_event_sent[method_name])) if method_name in state.task_event_started: - print "%s[%s]: %d" % (method_name, 'running', state.task_event_started[method_name]) + print("%s[%s]: %d" % (method_name, 'running', state.task_event_started[method_name])) if method_name in state.task_event_succeeded: - print "%s[%s]: %d" % (method_name, 'completed', state.task_event_succeeded[method_name]) + print("%s[%s]: %d" % (method_name, 'completed', state.task_event_succeeded[method_name])) if method_name in state.task_event_failed: - print "%s[%s]: %d" % (method_name, 'failed', state.task_event_failed[method_name]) + print("%s[%s]: %d" % (method_name, 'failed', state.task_event_failed[method_name])) num_waiting_by_task, num_running_by_task = state.num_waiting_running_by_task() - print '' - print 'Queue Sizes' - print 'Waiting Tasks: %d' % sum(num_waiting_by_task.values()) - print 'Running Tasks: %d' % sum(num_running_by_task.values()) + print('') + print('Queue Sizes') + print('Waiting Tasks: %d' % sum(num_waiting_by_task.values())) + print('Running Tasks: %d' % sum(num_running_by_task.values())) - print '' - print '' + print('') + print('') diff --git a/celery_cloudwatch/state.py b/celery_cloudwatch/state.py index a812539..0a0d092 100644 --- a/celery_cloudwatch/state.py +++ b/celery_cloudwatch/state.py @@ -32,10 +32,10 @@ def freeze_while(self, fun, *args, **kwargs): try: return fun(*args, **kwargs) except: - print "Exception in user code:" - print '-'*60 + print("Exception in user code:") + print('-'*60) traceback.print_exc(file=sys.stdout) - print '-'*60 + print('-'*60) finally: if clear_after: self._clear() diff --git a/celery_cloudwatch/task_monitor.py b/celery_cloudwatch/task_monitor.py index 4d6be19..7c83b0f 100644 --- a/celery_cloudwatch/task_monitor.py +++ b/celery_cloudwatch/task_monitor.py @@ -46,6 +46,6 @@ def proxy_event(self, event_name, fn): return fn else: def proxy_event_fn(event): - print '[{}] - {}'.format(event_name, pprint.pformat(event)) + print('[{}] - {}'.format(event_name, pprint.pformat(event))) return fn(event) return proxy_event_fn diff --git a/setup.py b/setup.py index 836479b..c09b345 100755 --- a/setup.py +++ b/setup.py @@ -9,9 +9,6 @@ needs_pytest = {'pytest', 'test', 'ptr'}.intersection(sys.argv) pytest_runner = ['pytest-runner'] if needs_pytest else [] -needs_setupext_pip = {'requirements'}.intersection(sys.argv) -setupext_pip = ['setupext-pip~=1.0.5'] if needs_setupext_pip else [] - here = os.path.abspath(os.path.dirname(__file__)) @@ -37,6 +34,7 @@ def read_markdown(*file_paths): except ImportError: return '' + setup( name='celery-cloudwatch', version=find_version("VERSION"),