forked from treeverse/dvc
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathgs.py
More file actions
42 lines (31 loc) 路 1.1 KB
/
Copy pathgs.py
File metadata and controls
42 lines (31 loc) 路 1.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
import base64
import threading
from funcy import cached_property, wrap_prop
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
from .fsspec_wrapper import ObjectFSWrapper
# pylint:disable=abstract-method
class GSFileSystem(ObjectFSWrapper):
scheme = Schemes.GS
PATH_CLS = CloudURLInfo
REQUIRES = {"gcsfs": "gcsfs"}
PARAM_CHECKSUM = "etag"
DETAIL_FIELDS = frozenset(("etag", "size"))
def __init__(self, **config):
super().__init__(**config)
url = config.get("url", "gs://")
self.path_info = self.PATH_CLS(url)
def _prepare_credentials(self, **config):
login_info = {"consistency": None}
login_info["project"] = config.get("projectname")
login_info["token"] = config.get("credentialpath")
return login_info
def _entry_hook(self, entry):
if "etag" in entry:
entry["etag"] = base64.b64decode(entry["etag"]).hex()
return entry
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
from gcsfs import GCSFileSystem
return GCSFileSystem(**self.fs_args)