Skip to content
This repository has been archived by the owner on Sep 8, 2022. It is now read-only.

Commit

Permalink
fixed some corner cases with read chunking changes
Browse files Browse the repository at this point in the history
  • Loading branch information
dweaver committed Jun 9, 2014
1 parent 4bf2782 commit 026e9f5
Show file tree
Hide file tree
Showing 6 changed files with 53 additions and 42 deletions.
7 changes: 5 additions & 2 deletions HISTORY.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
History
=======

0.7.15 (2014-06-05)
-------------------
0.8.0 (2014-06-09)
------------------

- transform command for modifying time series data
- EXO_PLUGIN_PATH variable to specify additional places to look for plugins
- spec support for domain level schema updates
- (breaking change) remove --chunkhours options and add --chunksize option,
which usually does not need to be specified
- fixed Python 3.4 regression

0.7.14 (2014-05-30)
-------------------
Expand Down
2 changes: 1 addition & 1 deletion exoline/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '0.7.15'
__version__ = '0.8.0'
30 changes: 19 additions & 11 deletions exoline/exo.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,8 @@
--timeformat=unix|human|iso8601
unix timestamp or human-readable? [default: human]
--header=name|rid include a header row
--chunksize=<size> break read into multiple requests of length
<size>, printing data as it is received.
[default: 212]
--chunksize=<size> [default: 212] break read into requests of
length <size>, printing data as it is received.
{{ helpoption }}
If <rid> is omitted, reads all datasources and datarules under <cik>.
Expand Down Expand Up @@ -365,17 +364,20 @@
else:
# plugin support for Windows executable build
try:
# spec plugin
try:
from ..exoline.plugins import spec
from ..exoline.plugins import transform
except:
from exoline.plugins import spec
from exoline.plugins import transform
# spec plugin
p = spec.Plugin()
plugins.append(p)
cmd_doc[p.command()] = spec.__doc__

# transform plugin
try:
from ..exoline.plugins import transform
except:
from exoline.plugins import transform
p = transform.Plugin()
plugins.append(p)
cmd_doc[p.command()] = transform.__doc__
Expand Down Expand Up @@ -607,10 +609,13 @@ def _read(cik, rids, options):
yield r
else:
# Read chunks by limit.
maxLimit = option['limit']
maxLimit = options['limit']
if 'sort' in options and options['sort'] == 'desc':
# descending
nextStart = options['endtime']
if 'endtime' in options:
nextStart = options['endtime']
else:
nextStart = ExoUtilities.parse_ts_tuple(datetime.now().timetuple())
while True:
chunkOpt = options.copy()
chunkOpt['endtime'] = nextStart
Expand All @@ -627,7 +632,10 @@ def _read(cik, rids, options):
yield r
else:
# ascending
nextStart = options['starttime']
if 'starttime' in options:
nextStart = options['starttime']
else:
nextStart = 0
while True:
chunkOpt = options.copy()
chunkOpt['starttime'] = nextStart
Expand All @@ -637,7 +645,7 @@ def _read(cik, rids, options):
break
maxLimit = maxLimit - len(res)
if maxLimit <= 0:
break;
break
#save oldest
nextStart = res[-1][0] + 1
for r in res:
Expand Down Expand Up @@ -1824,7 +1832,7 @@ def stripcarriage(s):

time.sleep(sleep_seconds)
else:
chunksize = args['--chunksize']
chunksize = int(args['--chunksize'])
result = er.readmult(cik,
rids,
sort=args['--sort'],
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
docopt==0.6.1
requests==1.2.3
requests==2.3.0
pyonep>=0.8.4
python-dateutil==2.1
pyyaml==3.10
Expand Down
50 changes: 25 additions & 25 deletions test/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -977,10 +977,10 @@ def read_test(self):
'--limit={0}'.format(numpts),
'--start={0}'.format(start),
'--end={0}'.format(end)]
r = rpc(*readcmd)
r = rpc(*(readcmd + ['--chunksize={0}'.format(numpts)]))
self.notok(r, "read a lot of data with a single big read")

readcmdchunks = readcmd + ['--chunkhours=6']
readcmdchunks = readcmd + ['--chunksize=512']
r = rpc(*readcmdchunks)
self.ok(r, "read a lot of data with multiple reads")

Expand Down Expand Up @@ -1297,7 +1297,7 @@ def spec_multi_test(self):
def spec_domain_test(self):
'''
Tests the ability to update multiple device of the same clientmodel
under multiple portals that are under a single domain.
under multiple portals that are under a single domain.
'''
# Get example spec
r = rpc('spec', '--example')
Expand All @@ -1309,74 +1309,74 @@ def spec_domain_test(self):
else:
out = r.stdout.encode('utf8')
f.write(out)


cik = self.client.cik()

# Create 2 users
portalOne_r = Resource(
cik,
'client',
{"name": "[email protected]"})

portalTwo_r = Resource(
cik,
'client',
{"name": "[email protected]"})

# Create two portals
portalOne = self._create(portalOne_r)
portalTwo = self._create(portalTwo_r)

# meta fields for test devices
metaMyModel = "{\"device\":{\"type\":\"vendor\",\"model\":\"myModel\",\"vendor\":\"myVendor\"}}"
metaNotMyModel = "{\"device\":{\"type\":\"vendor\",\"model\":\"NotMyModel\",\"vendor\":\"myVendor\"}}"


# Create two devices of myModel type,
myDev1_r = Resource(
portalOne.cik(),
'client',
{"name": "myDev1",
"meta":metaMyModel})

myDev2_r = Resource(
portalOne.cik(),
'client',
{"name": "myDev2",
"meta":metaMyModel})

# Create one device of notMyModel type
notMyDev_r = Resource(
portalOne.cik(),
'client',
{"name": "notMyDev",
"meta":metaNotMyModel})

# Create once device without a model type
genericDev_r = Resource(
portalOne.cik(),
'client',
{"name": "genericDev"})

# Create devices under each portal
p1_myDev1 = self._create(myDev1_r)
p1_myDev2 = self._create(myDev2_r)
p1_notMyDev = self._create(notMyDev_r)
p1_genericDev = self._create(genericDev_r)

# set parent cik of resources to portal 2
myDev1_r.parentcik = portalTwo.cik()
myDev2_r.parentcik = portalTwo.cik()
notMyDev_r.parentcik = portalTwo.cik()
genericDev_r.parentcik = portalTwo.cik()

# create devices under portal 2
p2_myDev1 = self._create(myDev1_r)
p2_myDev2 = self._create(myDev2_r)
p2_notMyDev = self._create(notMyDev_r)
p2_genericDev = self._create(genericDev_r)

# Portal 1
# Attempt to apply spec to myModel types
r = rpc('spec', cik, example_spec, '--domain', '-f', '--create', '--update-scripts', '--ids=A,B')
Expand All @@ -1385,34 +1385,34 @@ def spec_domain_test(self):
# make sure that both myDevs now meet spec
r = rpc('spec', p1_myDev1.cik(), example_spec, '--ids=A,B')
self.ok(r, "P1 Device 1 didn't match spec", search='')

r = rpc('spec', p1_myDev2.cik(), example_spec, '--ids=A,B')
self.ok(r, "P1 Device 2 didn't match spec", search='')
# and that both the non-example or the one that didn't have

# and that both the non-example or the one that didn't have
# a type don't meet the spec.
r = rpc('spec', p1_notMyDev.cik(), example_spec, '--ids=A,B')
self.ok(r, "P1 Device didn't match spec", search='not found')

r = rpc('spec', p1_genericDev.cik(), example_spec, '--ids=A,B')
self.ok(r, "P1 Device didn't match spec", search='not found')

# Portal 2
# make sure that both myDevs now meet spec
r = rpc('spec', p2_myDev1.cik(), example_spec, '--ids=A,B')
self.ok(r, "P2 Device 1 didn't match spec", search='')

r = rpc('spec', p2_myDev2.cik(), example_spec, '--ids=A,B')
self.ok(r, "P2 Device 2 didn't match spec", search='')
# and that both the non-example or the one that didn't have

# and that both the non-example or the one that didn't have
# a type don't meet the spec.
r = rpc('spec', p2_notMyDev.cik(), example_spec, '--ids=A,B')
self.ok(r, "P2 Device didn't match spec", search='not found')

r = rpc('spec', p2_genericDev.cik(), example_spec, '--ids=A,B')
self.ok(r, "P2 Device didn't match spec", search='not found')

def portals_cache_test(self):
'''Portals clearcache command and option'''
cik = self.client.cik()
Expand Down
4 changes: 2 additions & 2 deletions test/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ if [ "$1" == "full" ];
then
#set -e
# Python versions to test
#declare -a pythons=('python2.6' 'python2.7' 'python3.2' 'python3.3' 'python3.4')
declare -a pythons=('python2.7')
declare -a pythons=('python2.6' 'python2.7' 'python3.2' 'python3.3' 'python3.4')
#declare -a pythons=('python2.6')

for i in "${pythons[@]}"
do
Expand Down

0 comments on commit 026e9f5

Please sign in to comment.