I figured it out. It took a few tries. Once I figured out how to go back byte by byte, I did the math and saw where everything was located. The end of the document has a disproportionate number of bytes than the lines. And apparently line break is one byte (and not physically written as \n?).
I still have to clean up the code and once the screen is how I want it, I will implement the file naming system.
import logging
from waveshare_epd import epd7in5bc
import time
from PIL import Image,ImageDraw,ImageFont
import traceback
from datetime import date
from datetime import datetime
#Date and Time
today = date.today()
makeMonth = today.strftime("%b")
makeDay = today.strftime("%d")
now = datetime.now()
currentTime = now.strftime("%H")
day = today.weekday()
#Current Week
mm = [] #Month, day, daycode, food, med 1, med 2, dumped data
tum = []
wm = []
thm = []
fm = []
sam = []
sunm = []
dnumb = ['mm','tum', 'wm', 'thm', 'fm', 'sam', 'sunm']
# Pull data from file and write to lists
with open('list.txt', 'rb') as f:
f.seek(-2, os.SEEK_END)
while f.read(1) != b'\n':
f.seek(-2, os.SEEK_CUR)
last_line = f.readline().decode()
ps = last_line.split('*')
ds = ps[2]
dsms = dnumb[int(ds)]
eval(dsms).extend(ps)
loc = (f.tell() -29)
# Grab 4 more days
Counttm = 0
while Counttm < 4:
f.seek(loc, os.SEEK_SET)
Lline = f.readline().decode()
pss = Lline.split('*')
ds = pss[2]
dsmss = dnumb[int(ds)]
eval(dsmss).extend(pss)
loc = loc -15
Counttm = Counttm +1
f.close()
# debug importing
print(mm)
print(tum)
print(wm)
print(thm)
print(fm)
print(sam)
print(sunm)
Thanks for all the help guys!
Here's a petevent.py defining a class for storing, retrieving, examining, and comparing events:
from struct import pack, unpack, calcsize
from datetime import datetime, timezone, date, time, timedelta
import os # For os.fsync()
# This file is in public domain, or licensed under CC0-1.0, whichever you prefer.
class PetEvent(bytes):
"""Class describing an identifiable event (0-255) at a specific time"""
def __new__(cls, *args, tz=None):
"""PetEvent(bytes)
Creates an event from 8 bytes of binary data
PetEvent(kind)
Creates an event of type 'kind' for this local date and time,
'kind' being between 0 and 255
PetEvent(kind, year, month, day, hour, minute, second [, tz])
Creates an event of type 'kind' for the specified date and time.
If tz is not specified, local timezone is used.
"""
# PetEvent(bytes)?
if len(args) == 1 and isinstance(args[0], (bytes, bytearray)):
if len(args[0]) < 8:
raise ValueError("Events need exactly 8 bytes, received %d." % len(args[0]))
else:
return bytes.__new__(cls, (args[0])[0:8])
elif len(args) < 1:
raise ValueError("Cannot create an event out of nothing.")
# First parameter must be the event kind.
if (not isinstance(args[0], int)) or (args[0] < 0) or (args[0] > 255):
raise ValueError("Invalid kind (%s); must be an integer between 0 and 255, inclusive." % str(type(args[0])))
# PetEvent(kind)?
if len(args) == 1:
now = datetime.utcnow()
return bytes.__new__(cls, pack('>H6B', now.year, now.month, now.day, now.hour, now.minute, now.second, args[0]))
# PetEvent(kind, year, month, day, hour, minute, second)?
if len(args) == 7:
# We'll let datetime handle the date and time verification.
if tz is None:
then = datetime(*args[1:7])
else:
then = datetime(*args[1:7], 0, tz)
return bytes.__new__(cls, pack('>H6B', then.year, then.month, then.day, then.hour, then.minute, then.second, args[0]))
raise ValueError("Cannot construct an event from %s." % str(args))
def __sub__(self, value):
"""Subtracting two events yields their date and time difference as a timedelta object."""
return self.datetime(tz=timezone.utc) - value.datetime(tz=timezone.utc)
def __str__(self):
"""String description of the event is 'YYYY-MM-DD HH:MM:SS UTC K', where K is the kind"""
return '%04d-%02d-%02d %02d:%02d:%02d UTC %d' % self.tuple
# Properties: tuple, localtuple, year, month, day, hour, minute, second, kind.
@property
def tuple(self):
"""Returns a tuple(year, month, day, hour, minute, second, kind), in UTC."""
return unpack('>H6B', self[0:8])
@property
def localtuple(self):
"""Returns a tuple(year, month, day, hour, minute, second, kind), in local time."""
data = unpack('>H6B', self[0:8])
then = datetime(*data[0:6], 0, timezone.utc)
return (then.year, then.month, then.day, then.hour, then.minute, then.second, data[6])
@property
def year(self):
return unpack('>H', self[0:2])[0]
@property
def month(self):
return unpack('>B', self[2:3])[0]
@property
def day(self):
return unpack('>B', self[3:4])[0]
@property
def hour(self):
return unpack('>B', self[4:5])[0]
@property
def minute(self):
return unpack('>B', self[5:6])[0]
@property
def second(self):
return unpack('>B', self[6:7])[0]
@property
def kind(self):
"""Event type (0 to 255, inclusive)"""
return unpack('>B', self[7:8])[0]
# date(), time(), and datetime() methods
def date(self, tz=None):
"""Event date as a datetime.date object (no time)"""
if tz is None:
return date(*unpack('>H3B', self[0:4]))
else:
return date(*unpack('>H3B', self[0:4]), tz)
def time(self, tz=None):
"""Event time as a datetime.time object (no date)"""
if tz is None:
return time(*unpack('>3B', self[4:7]), 0)
else:
return time(*unpack('>3B', self[4:7]), 0, tz)
def datetime(self, tz=None):
"""Event date and time as a datetime.datetime object"""
if tz is None:
return datetime(*unpack('>H5B', self[0:7]))
else:
return datetime(*unpack('>H5B', self[0:7]), 0, tz)
def save(self, filename, sync=True):
"""Append this event to a binary file.
By default, this will return only after the
changes have been written to storage. If you
don't want that, use sync=False.
"""
target = open(filename, 'ab')
target.write(self)
if sync:
target.flush()
try:
os.fsync(target.fileno())
except:
pass
target.close()
@classmethod
def load(cls, filename, latest=None, blocksize=-1, kind=None, kinds=None, before=None, after=None,
years=None, months=None, weeks=None, days=None, hours=None, minutes=None, seconds=None):
"""Load a sequence of events from a file.
If you are only interested in the latest N events,
specify latest=N. Then, the events are produced in time order.
You can specify a single kind of event to get (kind=),
or a list or iterable of kinds to get (kinds=).
If you are only interested in events up to a specific moment,
you can specify that (before=).
If you are only interested in events since a specific moment,
you can specify that too (after=).
Alternatively, you can specify the number of weeks (weeks=),
days (days=), hours (hours=), minutes (minutes=), or seconds
(seconds=) prior to this moment to return the events for.
If you want the events as a list, wrap the call around list().
For constrained memory situations, supply blocksize with
a reasonable power of two, say 16384. For better performance,
use say 2097152. Default (-1) is for Python to use a heuristic.
"""
# Prepare to filter for specific kinds of events only.
keep_kinds = b''
if kind is not None:
keep_kinds += pack('>B', kind)
if kinds is not None:
for k in kinds:
keep_kinds += pack('>B', k)
# If there are no kinds specified, return all.
if len(keep_kinds) < 1:
keep_kinds = None
# Keep only events on and before a specific date and time?
if before is not None:
minwhen = pack('>H5B', before.year, after.month, before.day,
before.hour, before.minute, before.second)
elif (years, months, weeks, days, hours, minutes, seconds).count(None) != 7:
when = datetime.utcnow()
if seconds is not None:
when -= timedelta(seconds=seconds)
if minutes is not None:
when -= timedelta(minutes=minutes)
if hours is not None:
when -= timedelta(hours=hours)
if days is not None:
when -= timedelta(days=days)
if weeks is not None:
when -= timedelta(weeks=weeks)
# Years are modified on calendar basis
if isinstance(years, int) and years >= 0:
y = years
elif years is not None:
raise ValueError("years must be an integer")
else:
y = 0
# As are months
if isinstance(months, int) and months >= 0:
m = months
elif months is not None:
raise ValueError("months must be an integer")
else:
m = 0
# Combine all 12 month periods into years
if m >= 12:
y = y + (m // 12) # // is integer division
m = m % 12
# Modify months first
if m != 0:
if m < when.month:
when.replace(month=when.month-m)
else:
# It'll be the previous year.
y = y + 1
when.replace(month=when.month+12-m)
# And finally years.
if y != 0:
when.replace(year=when.year-y)
# Now pack it up.
minwhen = pack('>H5B', when.year, when.month, when.day,
when.hour, when.minute, when.second)
# and remove the datetime object from the local namespace.
del when
else:
minwhen = b'\x00\x00\x00\x00\x00\x00\x00'
# Keep only events on and after a specific date and time?
if after is not None:
maxwhen = pack('>H5B', after.year, after.month, after.day,
after.hour, after.minute, after.second)
else:
maxwhen = b'\xFF\xFF\xFF\xFF\xFF\xFF\xFF'
# No events read yet.
events = []
# Ready to read the binary file in chunks.
with open(filename, 'rb', buffering=blocksize) as source:
while True:
chunk = source.read(8)
if len(chunk) == 0:
break
elif len(chunk) != 8:
# Make sure we read a full chunk! Could be a pipe.
while len(chunk) < 8:
more = source.read(8 - len(chunk))
if len(more) == 0:
raise IOError("Partial event in file!")
chunk += more
# We always filter by datetime.
# This comparison only works because of the order,
# and when using '>H5B' to encode it!
if (chunk[0:7] < minwhen) or (chunk[0:7] > maxwhen):
continue
# Filter by kind, if desired.
if (keep_kinds is not None) and (chunk[7:8] not in keep_kinds):
continue
if latest is None:
# Yield (as part of the sequence) this event.
yield cls(chunk)
else:
# Add to the list.
events.append(cls(chunk))
# Need to drop any?
if len(events) > latest:
# Drop the oldest event in the list.
events.sort()
events.pop(0)
if len(events) > 0:
# Make sure they are in ascending order.
events.sort()
for event in events:
yield event
If you save it as petevent.py, you can run pydoc3 petevent in the same directory to read its documentation.
To use it in your own scripts, just put it in the same directory as your script, and in the beginning of your own code, add from petevent import PetEvent
Each PetEvent has a date and time, and a kind between 0 and 255, inclusive. You can access these fields using .year, .month, .day, .hour, .minute, .second, and .kind properties. You can get these as a tuple in this order via the .tuple property, or via the .localtuple property if you want the date and time in local timezone. Substracting two PetEvents yields a datetime.timedelta object describing their difference in time; the kind is ignored. If you want the difference in seconds, use (ev1 - ev2).total_seconds().
In the binary file and in the binary event objects, the date and time are stored in UTC, to avoid issues with daylight savings.
The PetEvent itself is a subclass of bytes, and consist of 8-byte bytes objects, and functions to manipulate them. To ensure that they sort in time order (and in ascending kind for events at the same second), I used the >H6b packing. That is, the year part is more significant byte first.
To create an event, say of kind 1, and save it safely to a binary file named data.bin, you only need to do
PetEvent(1).save("data.bin")
The save() method opens and closes the file, and uses OS-specific fsync() if possible, to ensure the data hits the storage before the method returns. If you don't want it to do that, use
PetEvent(1).save("data.bin", sync=False)
instead.
In particular, you should not lose any data in the file if the Python process crashes. You shouldn't lose any data if the machine crashes either, but to be sure, I might add an fsync to the directory the data file is in. (The only issue with the machine crashing is if it crashes before the file metadata, including the size, is updated. The data will be on disk, but the metadata might not be, you see.)
You can also create an event for a specific date and time, by default using the local timezone, using
event = PetEvent(kind, year, month, day, hour, minute, second)
or similarly for this moment,
event = PetEvent(kind)
and compare them or subtract them to get their time difference in datetime.timedelta units. (Use (A - B).total_seconds() to get the number of seconds between PetEvents A and B.)
Let's say you save some test events into a file named data.bin.
To print all events in it in UTC, (YYYY-MM-DD HH:MM:SS UTC kind), you just do
for event in PetEvent.load("data.bin"):
print(event)
For formatted output in local timezone, use e.g.
for event in PetEvent.load("data.bin"):
fields = event.localtuple
print("%04d-%02d-%02d %02d:%02d:%02d kind %d" % fields)
The loader has pretty advanced filtering built in. You can load only events of a specific type by supplying kind=kind, or of some specific types using kinds=(one, another). You can use datetime.datetime objects to specify what events you are interested in, by supplying them as after=datetime and/or before=datetime. If you are interested in events in the last N years, months, weeks, days, hours, minutes, or seconds, just supply years=N, months=N, and so on. (You can use more than one, say months=2, days=4 for events in the last two months and four days, but not with before or after). If you only want the latest N, supply latest=N too.
The loader produces a sequence, so if you want the events as a list, wrap it inside list():
events = list(PetEvent.load("data.bin", latest=15, kind=3))
If you are only interested in how many events of say kind 5 and 3 have occurred in the last 24 hours, use len(list(PetEvent.load("data.bin", kinds=(3,5)))).
While the class isn't optimal, it does NOT read the entire file in memory: it is designed to work without using up too much memory. You can even pass blocksize=16384 to the loader functions, to ensure they don't use too much memory for file buffering, at the cost of more syscalls for very large files.
Python I/O isn't that fast, and on my Core i5-4200U laptop, reading 365000 events (a 2.8 MiB file) takes about half a second. Appending a new event takes very little time, and does not depend on the file size. (This is true for both reading them all into a list, or just the last N events. The only difference there is the amount of RAM used by the Python process.)
If you use e.g. Qt5 or any GUI toolkit, you can put the loader in a separate thread or process, and provide the data as a Queue to the main process. That way the main process stays responsive, and can show e.g. a "loading" icon, and display the latest events when they've been read from the file. Similarly, a thread/subprocess could be used to save the data, and provide a success message via a Queue to the QUI thread, while keeping the UI responsive. (That way, when the user presses a button, they'd see e.g. on a status line a "Saving.." icon, that turns to "Ok" whenever the data on disk and the metadata have been updated and sync'ed.)
Finally, I just wrote this code, and tested it only lightly; it needs a check-over by another pair of eyes, before I'd really trust it. I did test most of the features, and it does seem to work, I'm just a bit paranoid and refuse to trust even myself without double-checking the code. I do make a lot of errors... :)