From db356b99a7c3d28d52e88b468ed7eafcdce5eb3b Mon Sep 17 00:00:00 2001 From: Hiroyasu Nishiyama Date: Thu, 28 Jan 2016 22:41:30 +0900 Subject: [PATCH] Added new features --- flickr_cli.py | 123 +++++++++++++++++++++++++++++++++----------- flickr_up.py | 139 ++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 201 insertions(+), 61 deletions(-) diff --git a/flickr_cli.py b/flickr_cli.py index a1edcb9..4c09a22 100644 --- a/flickr_cli.py +++ b/flickr_cli.py @@ -3,7 +3,7 @@ import logging import os.path import imghdr - +import re def valid_img(f): """ @@ -12,6 +12,11 @@ def valid_img(f): :param f: str that indicates the file directory. :return: boolean """ + if not os.path.isfile(f) : + return False + if re.search(r'\.(jpg|gif|png|tiff|mp4|mp2|avi|wmv|mov|m4v)$', f, + flags=re.IGNORECASE) : + return True try: file_type = imghdr.what(f) supported_types = ['jpeg', 'gif', 'png'] @@ -120,23 +125,33 @@ def get_photoset_id(self, title): :param title: :return: """ - self.photoset_id = self.exists(title) or self.create(title) - + self.photoset_id = self.exists(title) + if self.photoset_id : + return False + self.photoset_id = self.create(title) + return True + + def add_photo(self, id) : + ret = self.flickr.photosets_addPhoto(photoset_id=self.photoset_id, + photo_id=id) + return ret + def add_photos(self): """Adds photo ids to photoset on Flickr.""" - return [self.flickr.photosets_addPhoto( - photoset_id=self.photoset_id, - photo_id=i) for i in self.photo_ids] + return [self.add_photo(i) for i in self.photo_ids] def __call__(self, title, ids, primary_photo_id=0): """Generates photoset based on information passed in call""" + if len(ids) == 0 : + return None self.primary_photo_id = primary_photo_id or ids[0] self.photo_ids = ids - self.photo_ids.remove(self.primary_photo_id) - self.get_photoset_id(title) + if self.get_photoset_id(title) : + self.photo_ids.remove(self.primary_photo_id) if self.photo_ids: response = self.add_photos() return response + return None class AbstractDirectoryUpload(object): @@ -151,12 +166,12 @@ def filter_directory_contents(self, d, f): """Return true for files we don't want in our list (directories for now)""" return os.path.isdir(os.path.join(d, f)) - def get_directory_contents(self, d): + def get_directory_contents(self, d, **kwargs): """Get list of all files in a directory. TODO: Maybe this is better as a generator?""" - self.files = [os.path.join(d, f) - for f in os.listdir(d) - if not self.filter_directory_contents(d, f)] + self.files = sorted([os.path.join(d, f) + for f in os.listdir(d) + if not self.filter_directory_contents(d, f)]) def prehook(self, **kwargs): pass @@ -164,19 +179,19 @@ def prehook(self, **kwargs): def posthook(self, **kwargs): pass - def upload(self): + def upload(self, **kwargs): raise NotImplementedError - def parse_response(self): + def parse_response(self, **kwargs): raise NotImplementedError def __call__(self, directory, **kwargs): self.directory = directory self.prehook(**kwargs) - self.get_directory_contents(self.directory) - self.upload() - self.parse_response() + self.get_directory_contents(self.directory, **kwargs) + self.upload(**kwargs) + self.parse_response(**kwargs) self.posthook(**kwargs) @@ -195,12 +210,23 @@ def __init__(self, flickr): self.create_photoset = Photoset(flickr) def filter_directory_contents(self, d, f): - return not valid_img(os.path.join(d, f)) + path = os.path.join(d, f) + return not valid_img(path) + + def open_output(self, path) : + if path != None : + try : + fd = open(path, 'w', 1) + return fd + except IOError : + pass + return None def prehook(self, tags, pset, **kwargs): self.ids = [] self.tags = ", ".join(tags) self.photoset_name = pset + self.log = self.open_output(kwargs.get('log')) def flickr_upload(self, f, **kwargs): """ @@ -209,28 +235,49 @@ def flickr_upload(self, f, **kwargs): :param kwargs: :return: """ - print "Uploading %s" % f - return self.flickr.upload(filename=f, tags=self.tags, - is_public=kwargs.get('is_public', 0), - is_family=kwargs.get('is_family', 0)) - - def upload(self): + self.index += 1 + print "Uploading %4d/%4d: %s" % (self.index, self.count, f) + try : + result = self.flickr.upload(filename=f, tags=self.tags, + is_public=kwargs.get('is_public', 0), + is_family=kwargs.get('is_family', 0)) + except Exception as e : + print "; upload failed: %s" % e + result = None + ok = (result != None) and (result.attrib['stat'] == 'ok') + if (not ok) and (result != None) : + print "; upload failed." + if self.log != None : + tag = "+" if ok else "-" + self.log.write('%s,%s\n' % (tag, f)) + return result + + def upload(self, **kwargs): + self.count = len(self.files) + self.index = 0 self.responses = [(self.flickr_upload(f, is_public=0, is_family=0), f) for f in self.files] - def parse_response(self): + def parse_response(self, **kwargs): """ Handles response from Flickr API. :return: """ - self.ids = [r.find("photoid").text for (r, f) in self.responses if r.attrib['stat'] == "ok"] - self.failed_uploads = [f for (r, f) in self.responses if r.attrib['stat'] != "ok"] + succ_list = [(r, f) for (r, f) in self.responses if (r != None) and (r.attrib['stat'] == "ok")] + fail_list = [(r, f) for (r, f) in self.responses if (r == None) or (r.attrib['stat'] != "ok")] + self.ids = [r.find("photoid").text for (r, _) in succ_list] + self.failed_uploads = [f for (_, f) in fail_list] self.successful_uploads_count = len(self.ids) self.failed_uploads_count = len(self.failed_uploads) def posthook(self, **kwargs): - self.create_photoset(self.photoset_name, self.ids) + try : + self.create_photoset(self.photoset_name, self.ids) + except Exception as e : + print "failed to create photoset %s: %s" % (e, self.photoset_name) self.handle_failed_uploads() - + if self.log != None : + self.log.close() + def handle_failed_uploads(self): pass @@ -254,3 +301,21 @@ def flickr_upload(self, f, **kwargs): :type f: str The path to a file.""" return super(FamilyDirectoryUpload, self).flickr_upload(f, is_public=0, is_family=1) + + +class ArgFilesCollector : + """collect files specified as argument.""" + def get_directory_contents(self, d, **kwargs) : + self.files = sorted(kwargs.get('files')) + +class DirectoryFilesFlickrUpload(ArgFilesCollector, DirectoryFlickrUpload) : + """Uploads specified files in a directory as "private" files.""" + pass + +class PublicDirectoryFilesUpload(ArgFilesCollector, PublicDirectoryUpload) : + """Uploads specified files in a directory as "Public" files.""" + pass + +class FamilyDirectoryFilesUpload(ArgFilesCollector, FamilyDirectoryUpload) : + """Uploads specified files in a directory as "Family-only" files.""" + pass diff --git a/flickr_up.py b/flickr_up.py index 020a86d..c4296b5 100755 --- a/flickr_up.py +++ b/flickr_up.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # encoding: utf8 +import re import sys import os from optparse import OptionParser @@ -14,9 +15,99 @@ secret = config.get('flickr', 'secret') +def read_items(path) : + items = [] + if os.path.exists(path) : + try : + fd = open(path) + items = [line.rstrip() for line in fd if re.search(r'^\S.*$', line) != None] + fd.close() + except IOError as e: + logging.warning(e) + return items + +def make_tags(dir, tags) : + set0 = set(tags or []) + set1 = set(read_items('%s/tags.txt' % dir)) + return set0.union(set1) + +def make_title(dir, pset) : + items = read_items('%s/title.txt' % dir) + if len(items) > 0 : + return items[0] + return pset + def photoset_default_title(d): - return os.path.basename(os.path.normpath(d)) - + name = os.path.basename(os.path.normpath(d)) + return make_title(d, name) + +def is_excluded(path) : + name = os.path.basename(path) + return (name == 'tags.txt') or (name == 'title.txt') + +def upload_dir_rec0(flickr, directory, depth, tags, photoset, options) : + tags0 = tags or make_tags(directory, options.tags) or "" + photoset0 = photoset or photoset_default_title(directory) + log = options.log + + try : + files = [('%s/%s' % (directory, x)) for x in os.listdir(directory)] + except OSError: + files = [] + regs = [x for x in files if (os.path.isfile(x) and not is_excluded(x))] + dirs = [x for x in files if os.path.isdir(x)] + files = None + + print directory, tags0, photoset0 + + upload = flickr_cli.DirectoryFilesFlickrUpload(flickr) + upload(directory=directory, files=regs, pset=photoset0, tags=tags0, + log=log) + for subdir in dirs : + upload_dir_rec0(flickr, subdir, depth +1, tags, photoset, options) + +def upload_dir_rec(flickr, options) : + directory = options.directory + if options.same_recursive : + tags = make_tags(directory, options.tags) or "" + photoset = options.photoset or photoset_default_title(directory) + else : + tags = None + photoset = None + + upload_dir_rec0(flickr, directory, 0, tags, photoset, options) + +def upload_dir(flickr, options) : + directory = options.directory + log = options.log + tags = make_tags(directory, options.tags) or "" + photoset = options.photoset or photoset_default_title(directory) + + print directory, tags, photoset + + upload = flickr_cli.DirectoryFlickrUpload(flickr) + upload(directory=directory, pset=photoset, tags=tags, + log=log) + +def divide_files_by_dir(files) : + dir2files = dict() + for file in files : + dir = os.path.dirname(file) + if dir not in dir2files : + dir2files[dir] = [] + dir2files[dir].append(file) + return dir2files.items() + +def upload_files(flickr, directory, files, options) : + tags = make_tags(directory, options.tags) or "" + photoset = options.photoset or photoset_default_title(directory) + log = options.log + + print files, tags, photoset + + upload = flickr_cli.DirectoryFilesFlickrUpload(flickr) + upload(directory=directory, files=files, pset=photoset, tags=tags, + log=log) parser = OptionParser(version="1.0") @@ -38,41 +129,19 @@ def photoset_default_title(d): help="don't print status messages to stdout") parser.add_option("-r", "--recursive", - action="store_false", dest="recursive", default=False, + action="store_true", dest="recursive", default=False, help="recursively copy all subdirectories to different photosets") parser.add_option("-R", "--RECURSIVE", - action="store_false", dest="same_recursive", default=False, + action="store_true", dest="same_recursive", default=False, help="recursively copy all subdirectories to the same photoset. \ Overrides -r.") -(options, args) = parser.parse_args() - -if args: - print "ERROR" - print "Aborting due to unrecognized arguments:\n {}".format("\n\t".join(args)) - sys.exit(0) - -if not options.directory: - print "ERROR" - print "You must pass a directory.\n (example goes here)" - sys.exit(0) - -directory = options.directory -tags = options.tags or "" -photoset = options.photoset or photoset_default_title(options.directory) +parser.add_option("-l", "--log", + dest="log", default=None, + help="output uploaded files log.") -if not options.photoset or not options.tags: - print "WARNING" - -if not options.photoset: - print "You did not pass a name for the photoset.\n" - print "We will use the photoset title '{}'".format(photoset) - -if not options.tags: - print "You did not pass any tags.\n" - -print directory, tags, photoset +(options, args) = parser.parse_args() flickr = flickrapi.FlickrAPI(api_key, secret) @@ -83,5 +152,11 @@ def photoset_default_title(d): verifier = unicode(raw_input('Verifier code: ')) flickr.get_access_token(verifier) -upload = flickr_cli.DirectoryFlickrUpload(flickr) -upload(directory=directory, pset=photoset, tags=tags) +if options.directory != None : + if options.recursive or options.same_recursive : + upload_dir_rec(flickr, options) + else : + upload_dir(flickr, options) +if args != None : + for (dir, files) in divide_files_by_dir(args) : + upload_files(flickr, dir, files, options)