Mercurial and Git clients can push and pull from this alias URL to interact with this repository. You can change to which repository an alias points by going to the Aliases link on the project page.
# object_store.py -- Object store for git objects# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>## This program is free software; you can redistribute it and/or# modify it under the terms of the GNU General Public License# as published by the Free Software Foundation; either version 2# or (at your option) a later version of the License.## This program is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the# GNU General Public License for more details.## You should have received a copy of the GNU General Public License# along with this program; if not, write to the Free Software# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,# MA 02110-1301, USA."""Git object store interfaces and implementation."""importerrnoimportitertoolsimportosimportstatimporttempfilefromdulwich.diff_treeimport(tree_changes,walk_trees,)fromdulwich.errorsimport(NotTreeError,)fromdulwich.fileimportGitFilefromdulwich.objectsimport(Commit,ShaFile,Tag,Tree,ZERO_SHA,hex_to_sha,sha_to_hex,hex_to_filename,S_ISGITLINK,object_class,)fromdulwich.packimport(Pack,PackData,iter_sha1,write_pack_header,write_pack_index_v2,write_pack_object,write_pack_objects,compute_file_sha,PackIndexer,PackStreamCopier,)INFODIR='info'PACKDIR='pack'classBaseObjectStore(object):"""Object store interface."""defdetermine_wants_all(self,refs):return[shafor(ref,sha)inrefs.iteritems()ifnotshainselfandnotref.endswith("^{}")andnotsha==ZERO_SHA]defiter_shas(self,shas):"""Iterate over the objects for the specified shas. :param shas: Iterable object with SHAs :return: Object iterator """returnObjectStoreIterator(self,shas)defcontains_loose(self,sha):"""Check if a particular object is present by SHA1 and is loose."""raiseNotImplementedError(self.contains_loose)defcontains_packed(self,sha):"""Check if a particular object is present by SHA1 and is packed."""raiseNotImplementedError(self.contains_packed)def__contains__(self,sha):"""Check if a particular object is present by SHA1. This method makes no distinction between loose and packed objects. """returnself.contains_packed(sha)orself.contains_loose(sha)@propertydefpacks(self):"""Iterable of pack objects."""raiseNotImplementedErrordefget_raw(self,name):"""Obtain the raw text for an object. :param name: sha for the object. :return: tuple with numeric type and object contents. """raiseNotImplementedError(self.get_raw)def__getitem__(self,sha):"""Obtain an object by SHA1."""type_num,uncomp=self.get_raw(sha)returnShaFile.from_raw_string(type_num,uncomp)def__iter__(self):"""Iterate over the SHAs that are present in this store."""raiseNotImplementedError(self.__iter__)defadd_object(self,obj):"""Add a single object to this object store. """raiseNotImplementedError(self.add_object)defadd_objects(self,objects):"""Add a set of objects to this object store. :param objects: Iterable over a list of objects. """raiseNotImplementedError(self.add_objects)deftree_changes(self,source,target,want_unchanged=False):"""Find the differences between the contents of two trees :param source: SHA1 of the source tree :param target: SHA1 of the target tree :param want_unchanged: Whether unchanged files should be reported :return: Iterator over tuples with (oldpath, newpath), (oldmode, newmode), (oldsha, newsha) """forchangeintree_changes(self,source,target,want_unchanged=want_unchanged):yield((change.old.path,change.new.path),(change.old.mode,change.new.mode),(change.old.sha,change.new.sha))defiter_tree_contents(self,tree_id,include_trees=False):"""Iterate the contents of a tree and all subtrees. Iteration is depth-first pre-order, as in e.g. os.walk. :param tree_id: SHA1 of the tree. :param include_trees: If True, include tree objects in the iteration. :return: Iterator over TreeEntry namedtuples for all the objects in a tree. """forentry,_inwalk_trees(self,tree_id,None):ifnotstat.S_ISDIR(entry.mode)orinclude_trees:yieldentrydeffind_missing_objects(self,haves,wants,progress=None,get_tagged=None):"""Find the missing objects required for a set of revisions. :param haves: Iterable over SHAs already in common. :param wants: Iterable over SHAs of objects to fetch. :param progress: Simple progress function that will be called with updated progress strings. :param get_tagged: Function that returns a dict of pointed-to sha -> tag sha for including tags. :return: Iterator over (sha, path) pairs. """finder=MissingObjectFinder(self,haves,wants,progress,get_tagged)returniter(finder.next,None)deffind_common_revisions(self,graphwalker):"""Find which revisions this store has in common using graphwalker. :param graphwalker: A graphwalker object. :return: List of SHAs that are in common """haves=[]sha=graphwalker.next()whilesha:ifshainself:haves.append(sha)graphwalker.ack(sha)sha=graphwalker.next()returnhavesdefget_graph_walker(self,heads):"""Obtain a graph walker for this object store. :param heads: Local heads to start search with :return: GraphWalker object """returnObjectStoreGraphWalker(heads,lambdasha:self[sha].parents)defgenerate_pack_contents(self,have,want,progress=None):"""Iterate over the contents of a pack file. :param have: List of SHA1s of objects that should not be sent :param want: List of SHA1s of objects that should be sent :param progress: Optional progress reporting method """returnself.iter_shas(self.find_missing_objects(have,want,progress))defpeel_sha(self,sha):"""Peel all tags from a SHA. :param sha: The object SHA to peel. :return: The fully-peeled SHA1 of a tag object, after peeling all intermediate tags; if the original ref does not point to a tag, this will equal the original SHA1. """obj=self[sha]obj_class=object_class(obj.type_name)whileobj_classisTag:obj_class,sha=obj.objectobj=self[sha]returnobjclassPackBasedObjectStore(BaseObjectStore):def__init__(self):self._pack_cache=None@propertydefalternates(self):return[]defcontains_packed(self,sha):"""Check if a particular object is present by SHA1 and is packed."""forpackinself.packs:ifshainpack: return True
return False
+ def contains_alternate(self, sha):+ """Check if a particular object is present by SHA1 in the alternate storage."""+ for alternate in self.alternates:+ if alternate.contains_loose(sha) or alternate.contains_packed(sha):+ return True+ return False++ def __contains__(self, sha):+ """Check if a particular object is present by SHA1 in the main or alternate stores.++ This method makes no distinction between loose and packed objects.+ """+ return self.contains_packed(sha) or self.contains_loose(sha) or self.contains_alternate(sha)+ def _load_packs(self):
raise NotImplementedError(self._load_packs)
def_pack_cache_stale(self):"""Check whether the pack cache is stale."""raiseNotImplementedError(self._pack_cache_stale)def_add_known_pack(self,pack):"""Add a newly appeared pack to the cache by path. """ifself._pack_cacheisnotNone:self._pack_cache.append(pack)@propertydefpacks(self):"""List with pack objects.""" if self._pack_cache is None or self._pack_cache_stale():
self._pack_cache = self._load_packs()
return self._pack_cache
++ def _iter_alternate_objects(self):+ """Iterate over the SHAs of all the objects in alternate stores."""+ for alternate in self.alternates:+ for alternate_object in alternate:+ yield alternate_object def _iter_loose_objects(self):
"""Iterate over the SHAs of all loose objects."""
raiseNotImplementedError(self._iter_loose_objects)def_get_loose_object(self,sha):raiseNotImplementedError(self._get_loose_object)def_remove_loose_object(self,sha):raiseNotImplementedError(self._remove_loose_object)defpack_loose_objects(self):"""Pack loose objects. :return: Number of objects packed """objects=set()forshainself._iter_loose_objects():objects.add((self._get_loose_object(sha),None))self.add_objects(list(objects))forobj,pathinobjects:self._remove_loose_object(obj.id)returnlen(objects) def __iter__(self):
"""Iterate over the SHAs that are present in this store."""
- iterables = self.packs + [self._iter_loose_objects()]
+ iterables = self.packs + [self._iter_loose_objects()] + [self._iter_alternate_objects()] return itertools.chain(*iterables)
def contains_loose(self, sha):
"""Check if a particular object is present by SHA1 and is loose."""returnself._get_loose_object(sha)isnotNonedefget_raw(self,name):"""Obtain the raw text for an object. :param name: sha for the object. :return: tuple with numeric type and object contents. """iflen(name)==40:sha=hex_to_sha(name)hexsha=nameeliflen(name)==20:sha=namehexsha=Noneelse:raiseAssertionError("Invalid object name %r"%name)forpackinself.packs:try:returnpack.get_raw(sha)exceptKeyError:passifhexshaisNone:hexsha=sha_to_hex(name)ret=self._get_loose_object(hexsha)ifretisnotNone:returnret.type_num,ret.as_raw_string()foralternateinself.alternates:try:returnalternate.get_raw(hexsha)exceptKeyError:passraiseKeyError(hexsha)defadd_objects(self,objects):"""Add a set of objects to this object store. :param objects: Iterable over objects, should support __len__. :return: Pack object of the objects written. """iflen(objects)==0:# Don't bother writing an empty pack filereturnf,commit=self.add_pack()write_pack_objects(f,objects)returncommit()classDiskObjectStore(PackBasedObjectStore):"""Git-style object store that exists on disk."""def__init__(self,path):"""Open an object store. :param path: Path of the object store. """super(DiskObjectStore,self).__init__()self.path=pathself.pack_dir=os.path.join(self.path,PACKDIR)self._pack_cache_time=0self._alternates=None@propertydefalternates(self):ifself._alternatesisnotNone:returnself._alternatesself._alternates=[]forpathinself._read_alternate_paths():self._alternates.append(DiskObjectStore(path))returnself._alternatesdef_read_alternate_paths(self):try:f=GitFile(os.path.join(self.path,"info","alternates"),'rb')except(OSError,IOError),e:ife.errno==errno.ENOENT:return[]raiseret=[]try:forlinf.readlines():l=l.rstrip("\n")ifl[0]=="#":continueifnotos.path.isabs(l):continueret.append(l)returnretfinally:f.close()defadd_alternate_path(self,path):"""Add an alternate path to this object store. """try:os.mkdir(os.path.join(self.path,"info"))exceptOSError,e:ife.errno!=errno.EEXIST:raisealternates_path=os.path.join(self.path,"info/alternates")f=GitFile(alternates_path,'wb')try:try:orig_f=open(alternates_path,'rb')except(OSError,IOError),e:ife.errno!=errno.ENOENT:raiseelse:try:f.write(orig_f.read())finally:orig_f.close()f.write("%s\n"%path)finally:f.close()self.alternates.append(DiskObjectStore(path))def_load_packs(self):pack_files=[]try:self._pack_cache_time=os.stat(self.pack_dir).st_mtimepack_dir_contents=os.listdir(self.pack_dir)fornameinpack_dir_contents:# TODO: verify that idx exists firstifname.startswith("pack-")andname.endswith(".pack"):filename=os.path.join(self.pack_dir,name)pack_files.append((os.stat(filename).st_mtime,filename))exceptOSError,e:ife.errno==errno.ENOENT:return[]raisepack_files.sort(reverse=True)suffix_len=len(".pack")return[Pack(f[:-suffix_len])for_,finpack_files]def_pack_cache_stale(self):try:returnos.stat(self.pack_dir).st_mtime>self._pack_cache_timeexceptOSError,e:ife.errno==errno.ENOENT:returnTrueraisedef_get_shafile_path(self,sha):# Check from object dirreturnhex_to_filename(self.path,sha)def_iter_loose_objects(self):forbaseinos.listdir(self.path):iflen(base)!=2:continueforrestinos.listdir(os.path.join(self.path,base)):yieldbase+restdef_get_loose_object(self,sha):path=self._get_shafile_path(sha)try:returnShaFile.from_path(path)except(OSError,IOError),e:ife.errno==errno.ENOENT:returnNoneraisedef_remove_loose_object(self,sha):os.remove(self._get_shafile_path(sha))def_complete_thin_pack(self,f,path,copier,indexer):"""Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param f: Open file object for the pack. :param path: Path to the pack file. :param copier: A PackStreamCopier to use for writing pack data. :param indexer: A PackIndexer for indexing the pack. """entries=list(indexer)# Update the header with the new number of objects.f.seek(0)write_pack_header(f,len(entries)+len(indexer.ext_refs()))# Must flush before reading (http://bugs.python.org/issue3207)f.flush()# Rescan the rest of the pack, computing the SHA with the new header.new_sha=compute_file_sha(f,end_ofs=-20)# Must reposition before writing (http://bugs.python.org/issue3207)f.seek(0,os.SEEK_CUR)# Complete the pack.forext_shainindexer.ext_refs():assertlen(ext_sha)==20type_num,data=self.get_raw(ext_sha)offset=f.tell()crc32=write_pack_object(f,type_num,data,sha=new_sha)entries.append((ext_sha,offset,crc32))pack_sha=new_sha.digest()f.write(pack_sha)f.close()# Move the pack in.entries.sort()pack_base_name=os.path.join(self.pack_dir,'pack-'+iter_sha1(e[0]foreinentries))os.rename(path,pack_base_name+'.pack')# Write the index.index_file=GitFile(pack_base_name+'.idx','wb')try:write_pack_index_v2(index_file,entries,pack_sha)index_file.close()finally:index_file.abort()# Add the pack to the store and return it.final_pack=Pack(pack_base_name)final_pack.check_length_and_checksum()self._add_known_pack(final_pack)returnfinal_packdefadd_thin_pack(self,read_all,read_some):"""Add a new thin pack to this object store. Thin packs are packs that contain deltas with parents that exist outside the pack. They should never be placed in the object store directly, and always indexed and completed as they are copied. :param read_all: Read function that blocks until the number of requested bytes are read. :param read_some: Read function that returns at least one byte, but may not return the number of bytes requested. :return: A Pack object pointing at the now-completed thin pack in the objects/pack directory. """fd,path=tempfile.mkstemp(dir=self.path,prefix='tmp_pack_')f=os.fdopen(fd,'w+b')try:indexer=PackIndexer(f,resolve_ext_ref=self.get_raw)copier=PackStreamCopier(read_all,read_some,f,delta_iter=indexer)copier.verify()returnself._complete_thin_pack(f,path,copier,indexer)finally:f.close()defmove_in_pack(self,path):"""Move a specific file containing a pack into the pack directory. :note: The file should be on the same file system as the packs directory. :param path: Path to the pack file. """p=PackData(path)entries=p.sorted_entries()basename=os.path.join(self.pack_dir,"pack-%s"%iter_sha1(entry[0]forentryinentries))f=GitFile(basename+".idx","wb")try:write_pack_index_v2(f,entries,p.get_stored_checksum())finally:f.close()p.close()os.rename(path,basename+".pack")final_pack=Pack(basename)self._add_known_pack(final_pack)returnfinal_packdefadd_pack(self):"""Add a new pack to this object store. :return: Fileobject to write to and a commit function to call when the pack is finished. """fd,path=tempfile.mkstemp(dir=self.pack_dir,suffix=".pack")f=os.fdopen(fd,'wb')defcommit():os.fsync(fd)f.close()ifos.path.getsize(path)>0:returnself.move_in_pack(path)else:os.remove(path)returnNonereturnf,commitdefadd_object(self,obj):"""Add a single object to this object store. :param obj: Object to add """dir=os.path.join(self.path,obj.id[:2])try:os.mkdir(dir)exceptOSError,e:ife.errno!=errno.EEXIST:raisepath=os.path.join(dir,obj.id[2:])ifos.path.exists(path):return# Already there, no need to write againf=GitFile(path,'wb')try:f.write(obj.as_legacy_object())finally:f.close()@classmethoddefinit(cls,path):try:os.mkdir(path)exceptOSError,e:ife.errno!=errno.EEXIST:raiseos.mkdir(os.path.join(path,"info"))os.mkdir(os.path.join(path,PACKDIR))returncls(path)classMemoryObjectStore(BaseObjectStore):"""Object store that keeps all objects in memory."""def__init__(self):super(MemoryObjectStore,self).__init__()self._data={}def_to_hexsha(self,sha):iflen(sha)==40:returnshaeliflen(sha)==20:returnsha_to_hex(sha)else:raiseValueError("Invalid sha %r"%(sha,))defcontains_loose(self,sha):"""Check if a particular object is present by SHA1 and is loose."""returnself._to_hexsha(sha)inself._datadefcontains_packed(self,sha):"""Check if a particular object is present by SHA1 and is packed."""returnFalsedef__iter__(self):"""Iterate over the SHAs that are present in this store."""returnself._data.iterkeys()@propertydefpacks(self):"""List with pack objects."""return[]defget_raw(self,name):"""Obtain the raw text for an object. :param name: sha for the object. :return: tuple with numeric type and object contents. """obj=self[self._to_hexsha(name)]returnobj.type_num,obj.as_raw_string()def__getitem__(self,name):returnself._data[self._to_hexsha(name)]def__delitem__(self,name):"""Delete an object from this store, for testing only."""delself._data[self._to_hexsha(name)]defadd_object(self,obj):"""Add a single object to this object store. """self._data[obj.id]=objdefadd_objects(self,objects):"""Add a set of objects to this object store. :param objects: Iterable over a list of objects. """forobj,pathinobjects:self._data[obj.id]=objclassObjectImporter(object):"""Interface for importing objects."""def__init__(self,count):"""Create a new ObjectImporter. :param count: Number of objects that's going to be imported. """self.count=countdefadd_object(self,object):"""Add an object."""raiseNotImplementedError(self.add_object)deffinish(self,object):"""Finish the import and write objects to disk."""raiseNotImplementedError(self.finish)classObjectIterator(object):"""Interface for iterating over objects."""defiterobjects(self):raiseNotImplementedError(self.iterobjects)classObjectStoreIterator(ObjectIterator):"""ObjectIterator that works on top of an ObjectStore."""def__init__(self,store,sha_iter):"""Create a new ObjectIterator. :param store: Object store to retrieve from :param sha_iter: Iterator over (sha, path) tuples """self.store=storeself.sha_iter=sha_iterself._shas=[]def__iter__(self):"""Yield tuple with next object and path."""forsha,pathinself.itershas():yieldself.store[sha],pathdefiterobjects(self):"""Iterate over just the objects."""foro,pathinself:yieldodefitershas(self):"""Iterate over the SHAs."""forshainself._shas:yieldshaforshainself.sha_iter:self._shas.append(sha)yieldshadef__contains__(self,needle):"""Check if an object is present. :note: This checks if the object is present in the underlying object store, not if it would be yielded by the iterator. :param needle: SHA1 of the object to check for """returnneedleinself.storedef__getitem__(self,key):"""Find an object by SHA1. :note: This retrieves the object from the underlying object store. It will also succeed if the object would not be returned by the iterator. """returnself.store[key]def__len__(self):"""Return the number of objects."""returnlen(list(self.itershas()))deftree_lookup_path(lookup_obj,root_sha,path):"""Look up an object in a Git tree. :param lookup_obj: Callback for retrieving object by SHA1 :param root_sha: SHA1 of the root tree :param path: Path to lookup :return: A tuple of (mode, SHA) of the resulting path. """tree=lookup_obj(root_sha)ifnotisinstance(tree,Tree):raiseNotTreeError(root_sha)returntree.lookup_path(lookup_obj,path)classMissingObjectFinder(object):"""Find the objects missing from another object store. :param object_store: Object store containing at least all objects to be sent :param haves: SHA1s of commits not to send (already present in target) :param wants: SHA1s of commits to send :param progress: Optional function to report progress to. :param get_tagged: Function that returns a dict of pointed-to sha -> tag sha for including tags. :param tagged: dict of pointed-to sha -> tag sha for including tags """def__init__(self,object_store,haves,wants,progress=None,get_tagged=None):haves=set(haves)self.sha_done=havesself.objects_to_send=set([(w,None,False)forwinwantsifwnotinhaves])self.object_store=object_storeifprogressisNone:self.progress=lambdax:Noneelse:self.progress=progressself._tagged=get_taggedandget_tagged()or{}defadd_todo(self,entries):self.objects_to_send.update([eforeinentriesifnote[0]inself.sha_done])defparse_tree(self,tree):self.add_todo([(sha,name,notstat.S_ISDIR(mode))forname,mode,shaintree.iteritems()ifnotS_ISGITLINK(mode)])defparse_commit(self,commit):self.add_todo([(commit.tree,"",False)])self.add_todo([(p,None,False)forpincommit.parents])defparse_tag(self,tag):self.add_todo([(tag.object[1],None,False)])defnext(self):whileTrue:ifnotself.objects_to_send:returnNone(sha,name,leaf)=self.objects_to_send.pop()ifshanotinself.sha_done:breakifnotleaf:o=self.object_store[sha]ifisinstance(o,Commit):self.parse_commit(o)elifisinstance(o,Tree):self.parse_tree(o)elifisinstance(o,Tag):self.parse_tag(o)ifshainself._tagged:self.add_todo([(self._tagged[sha],None,True)])self.sha_done.add(sha)self.progress("counting objects: %d\r"%len(self.sha_done))return(sha,name)classObjectStoreGraphWalker(object):"""Graph walker that finds what commits are missing from an object store. :ivar heads: Revisions without descendants in the local repo :ivar get_parents: Function to retrieve parents in the local repo """def__init__(self,local_heads,get_parents):"""Create a new instance. :param local_heads: Heads to start search with :param get_parents: Function for finding the parents of a SHA1. """self.heads=set(local_heads)self.get_parents=get_parentsself.parents={}defack(self,sha):"""Ack that a revision and its ancestors are present in the source."""ancestors=set([sha])# stop if we run out of heads to removewhileself.heads:forainancestors:ifainself.heads:self.heads.remove(a)# collect all ancestorsnew_ancestors=set()forainancestors:ps=self.parents.get(a)ifpsisnotNone:new_ancestors.update(ps)self.parents[a]=None# no more ancestors; stopifnotnew_ancestors:breakancestors=new_ancestorsdefnext(self):"""Iterate over ancestors of heads in the target."""ifself.heads:ret=self.heads.pop()ps=self.get_parents(ret)self.parents[ret]=psself.heads.update([pforpinpsifnotpinself.parents])returnretreturnNone
Attach a Trello Card
Add a tag
Your session has expired
You are no longer logged in. Please log in and try your request again.
Filter RSS Feed
This RSS feed URL allows you to see the contents of your current filter using any feed reader.
This link includes a special authentication token. If you share the URL with anyone else, they can see this RSS feed's activity. You can disable these tokens when needed.
Your current filter is unsaved; changing it won't affect this RSS feed.