Commit: 0966df4efa0db839ec8bc6b9ef9028bf8e264615 - allura (git)

allura

Commit MetaInfo

• Revisão: 0966df4efa0db839ec8bc6b9ef9028bf8e264615 (tree)
• Hora: 2011-10-06 02:14:55
• Autor: Rick Copeland <rcopeland@geek...>
• Commiter: Rick Copeland

Mensagem de Log

[#2020] Integrating new repo refresh code with existing repo models

Signed-off-by: Rick Copeland <[e-mail]>

Mudança Sumário

• modified: Allura/allura/model/index.py
• modified: Allura/allura/model/repo.py
• modified: Allura/allura/model/repository.py
• modified: Allura/test-light.py
• modified: ForgeGit/forgegit/model/git_repo.py

Diff

--- a/Allura/allura/model/index.py
+++ b/Allura/allura/model/index.py
@@ -2,43 +2,47 @@ import re
 import logging
 from itertools import groupby
 from cPickle import dumps, loads
-from datetime import datetime
 from collections import defaultdict
 
 import bson
 import pymongo
-from pylons import c, g
+from pylons import c
 
-import ming
+from ming import collection, Field, Index
 from ming import schema as S
 from ming.utils import LazyProperty
-from ming.orm import session
-from ming.orm import FieldProperty, ForeignIdProperty, RelationProperty
-from ming.orm.declarative import MappedClass
+from ming.orm import session, mapper
+from ming.orm import ForeignIdProperty, RelationProperty
 
 from allura.lib import helpers as h
 
-from .session import main_orm_session
+from .session import main_doc_session, main_orm_session
 
 log = logging.getLogger(__name__)
 
-class ArtifactReference(MappedClass):
-    '''ArtifactReference manages the artifact graph.
-
-    fields are all strs, corresponding to Solr index_ids
-    '''
-    class __mongometa__:
-        session = main_orm_session
-        name = 'artifact_reference'
-        indexes = [ 'references' ]
-
-    _id = FieldProperty(str)
-    artifact_reference = FieldProperty(S.Object(dict(
-            cls=S.Binary,
-            project_id=S.ObjectId,
-            app_config_id=S.ObjectId,
-            artifact_id=S.Anything(if_missing=None))))
-    references = FieldProperty([str])
+# Collection definitions
+ArtifactReferenceDoc = collection(
+    'artifact_reference', main_doc_session,
+    Field('_id', str),
+    Field('artifact_reference', dict(
+            cls=S.Binary(),
+            project_id=S.ObjectId(),
+            app_config_id=S.ObjectId(),
+            artifact_id=S.Anything(if_missing=None))),
+    Field('references', [str], index=True))
+
+ShortlinkDoc = collection(
+    'shortlink', main_doc_session,
+    Field('_id', S.ObjectId()),
+    Field('ref_id', str, index=True),
+    Field('project_id', S.ObjectId()),
+    Field('app_config_id', S.ObjectId()),
+    Field('link', str),
+    Field('url', str),
+    Index('link, project_id', 'app_config_id'))
+
+# Class definitions
+class ArtifactReference(object):
 
     @classmethod
     def from_artifact(cls, artifact):
@@ -71,28 +75,8 @@ class ArtifactReference(MappedClass):
             log.exception('Error loading artifact for %s: %r',
                           self._id, aref)
 
-class Shortlink(MappedClass):
+class Shortlink(object):
     '''Collection mapping shorthand_ids for artifacts to ArtifactReferences'''
-    class __mongometa__:
-        session = main_orm_session
-        name = 'shortlink'
-        indexes = [
-            ('link', 'project_id', 'app_config_id'),
-            ('ref_id',),
-            ]
-
-    # Stored properties
-    _id = FieldProperty(S.ObjectId)
-    ref_id = ForeignIdProperty(ArtifactReference)
-    project_id = ForeignIdProperty('Project')
-    app_config_id = ForeignIdProperty('AppConfig')
-    link = FieldProperty(str)
-    url = FieldProperty(str)
-
-    # Relation Properties
-    project = RelationProperty('Project')
-    app_config = RelationProperty('AppConfig')
-    ref = RelationProperty('ArtifactReference')
 
     # Regexes used to find shortlinks
     _core_re = r'''(\[
@@ -202,3 +186,12 @@ class Shortlink(MappedClass):
         else:
             return None
 
+# Mapper definitions
+mapper(ArtifactReference, ArtifactReferenceDoc, main_orm_session)
+mapper(Shortlink, ShortlinkDoc, main_orm_session, properties=dict(
+    ref_id = ForeignIdProperty(ArtifactReference),
+    project_id = ForeignIdProperty('Project'),
+    app_config_id = ForeignIdProperty('AppConfig'),
+    project = RelationProperty('Project'),
+    app_config = RelationProperty('AppConfig'),
+    ref = RelationProperty(ArtifactReference)))

--- a/Allura/allura/model/repo.py
+++ b/Allura/allura/model/repo.py
@@ -1,77 +1,59 @@
+import re
+import sys
+import logging
+from hashlib import sha1
+from itertools import izip, chain
 from datetime import datetime
+from collections import defaultdict
 
-from ming import Document, Field
+from pylons import g
+
+from ming import Field, Index, collection
 from ming import schema as S
+from ming.utils import LazyProperty
+from ming.orm import mapper
 
-from .session import main_doc_session, project_doc_session
+from allura.lib import utils
+from allura.lib import helpers as h
 
-class Commit(Document):
-    class __mongometa__:
-        name = 'repo_ci'
-        session = main_doc_session
-        indexes = [
-            ('parent_ids',),
-            ('child_ids',),
-            ('repo_ids',)]
-    User = dict(name=str, email=str, date=datetime)
-
-    _id = Field(str)
-    tree_id = Field(str)
-    committed = Field(User)
-    authored = Field(User)
-    message = Field(str)
-    parent_ids = Field([str])
-    child_ids = Field([str])
-    repo_ids = Field([S.ObjectId()])
+from .auth import User
+from .session import main_doc_session, project_doc_session
+from .session import repository_orm_session
 
-    def __repr__(self):
-        return '%s %s' % (
-            self._id[:7], self.summary)
+log = logging.getLogger(__name__)
 
-    @property
-    def summary(self):
-        if self.message:
-            summary = []
-            for line in self.message.splitlines():
-                line = line.rstrip()
-                if line: summary.append(line)
-                else: return ' '.join(summary)
-            return ' '.join(summary)
-        return ''
+SUser = dict(name=str, email=str, date=datetime)
+SObjType=S.OneOf('blob', 'tree', 'submodule')
+QSIZE = 100
+README_RE = re.compile('^README(\.[^.]*)?$', re.IGNORECASE)
 
-    def url(self):
-        return ''
+# Basic commit information
+CommitDoc = collection(
+    'repo_ci', main_doc_session,
+    Field('_id', str),
+    Field('tree_id', str),
+    Field('committed', SUser),
+    Field('authored', SUser),
+    Field('message', str),
+    Field('parent_ids', [str], index=True),
+    Field('child_ids', [str], index=True),
+    Field('repo_ids', [ S.ObjectId() ], index=True))
 
-    def shorthand_id(self):
-        return ''
+# Basic tree information
+TreeDoc = collection(
+    'repo_tree', main_doc_session,
+    Field('_id', str),
+    Field('tree_ids', [dict(name=str, id=str)]),
+    Field('blob_ids', [dict(name=str, id=str)]),
+    Field('other_ids', [dict(name=str, id=str, type=SObjType)]))
 
-    @property
-    def author_url(self):
-        return ''
-
-class Tree(Document):
-    class __mongometa__:
-        name = 'repo_tree'
-        session = main_doc_session
-    ObjType=S.OneOf('blob', 'tree', 'submodule')
-
-    _id = Field(str)
-    tree_ids = Field([dict(name=str, id=str)])
-    blob_ids = Field([dict(name=str, id=str)])
-    other_ids = Field([dict(name=str, id=str, type=ObjType)])
-
-class LastCommit(Document):
-    class __mongometa__:
-        name = 'repo_last_commit'
-        session = project_doc_session
-        indexes = [
-            ( 'repo_id', 'object_id'),
-            ]
-
-    _id = Field(str)
-    repo_id=Field(S.ObjectId())
-    object_id=Field(str)
-    commit_info = Field(dict(
+# Information about the last commit to touch a tree/blob
+LastCommitDoc = collection(
+    'repo_last_commit', project_doc_session,
+    Field('_id', str),
+    Field('repo_id', S.ObjectId()),
+    Field('object_id', str),
+    Field('commit_info', dict(
         id=str,
         date=datetime,
         author=str,
@@ -79,58 +61,298 @@ class LastCommit(Document):
         author_url=str,
         href=str,
         shortlink=str,
-        summary=str))
-
-    @classmethod
-    def set_last_commit(cls, repo_id, oid, commit):
-        lc = cls(dict(
-                _id='%s:%s' % (repo_id, oid),
-                repo_id=repo_id,
-                object_id=oid,
-                commit_info=dict(
-                    id=commit._id,
-                    author=commit.authored.name,
-                    author_email=commit.authored.email,
-                    author_url=commit.author_url,
-                    date=commit.authored.date,
-                    href=commit.url(),
-                    shortlink=commit.shorthand_id(),
-                    summary=commit.summary)))
-        lc.m.save(safe=False)
-        return lc
-
-class Trees(Document):
-    class __mongometa__:
-        name = 'repo_trees'
-        session = main_doc_session
-
-    _id = Field(str) # commit ID
-    tree_ids = Field([str]) # tree IDs
-
-class DiffInfo(Document):
-    class __mongometa__:
-        name = 'repo_diffinfo'
-        session = main_doc_session
-
-    _id = Field(str)
-    differences = Field([dict(name=str, lhs_id=str, rhs_id=str)])
-
-class BasicBlock(Document):
-    class __mongometa__:
-        name = 'repo_basic_block'
-        session = main_doc_session
-        indexes = [
-            ('commit_ids',) ]
-
-    _id = Field(str)
-    parent_commit_ids = Field([str])
-    commit_ids = Field([str])
-    commit_times = Field([datetime])
+        summary=str)),
+    Index('repo_id', 'object_id'))
+
+# List of all trees contained within a commit
+TreesDoc = collection(
+    'repo_trees', main_doc_session,
+    Field('_id', str),
+    Field('tree_ids', [str]))
+
+# Information about which things were added/removed in  commit
+DiffInfoDoc = collection(
+    'repo_diffinfo', main_doc_session,
+    Field('_id', str),
+    Field(
+        'differences',
+        [ dict(name=str, lhs_id=str, rhs_id=str)]))
+
+# List of commit runs (a run is a linear series of single-parent commits)
+CommitRunDoc = collection(
+    'repo_commitrun', main_doc_session,
+    Field('_id', str),
+    Field('parent_commit_ids', [str]),
+    Field('commit_ids', [str], index=True),
+    Field('commit_times', [datetime]))
+
+class RepoObject(object):
 
     def __repr__(self):
-        return '%s: (P %s, T %s..%s (%d commits))' % (
-            self._id[:6],
-            [ oid[:6] for oid in self.parent_commit_ids ],
-            self.commit_ids[0][:6],
-            self.commit_ids[-1][:6],
-            len(self.commit_ids))
+        return '<%s %s>' % (
+            self.__class__.__name__, self._id)
+
+    def primary(self):
+        return self
+
+    def index_id(self):
+        '''Globally unique artifact identifier.  Used for
+        SOLR ID, shortlinks, and maybe elsewhere
+        '''
+        id = '%s.%s#%s' % (
+            self.__class__.__module__,
+            self.__class__.__name__,
+            self._id)
+        return id.replace('.', '/')
+
+class Commit(RepoObject):
+    # Ephemeral attrs
+    repo=None
+
+    @LazyProperty
+    def author_url(self):
+        u = User.by_email_address(self.authored.email)
+        if u: return u.url()
+
+    @LazyProperty
+    def committer_url(self):
+        u = User.by_email_address(self.committed.email)
+        if u: return u.url()
+
+    @LazyProperty
+    def tree(self):
+        if self.tree_id is None:
+            self.tree_id = self.repo.compute_tree(self)
+        if self.tree_id is None:
+            return None
+        t = Tree.query.get(object_id=self.tree_id)
+        if t is None:
+            self.tree_id = self.repo.compute_tree(self)
+            t = Tree.query.get(object_id=self.tree_id)
+        if t is not None: t.set_context(self)
+        return t
+
+    @LazyProperty
+    def summary(self):
+        message = h.really_unicode(self.message)
+        first_line = message.split('\n')[0]
+        return h.text.truncate(first_line, 50)
+
+    def get_path(self, path):
+        '''Return the blob on the given path'''
+        if path.startswith('/'): path = path[1:]
+        path_parts = path.split('/')
+        return self.tree.get_blob(path_parts[-1], path_parts[:-1])
+
+    def shorthand_id(self):
+        return self.repo.shorthand_for_commit(self)
+
+    @LazyProperty
+    def symbolic_ids(self):
+        return self.repo.symbolics_for_commit(self)
+
+    def url(self):
+        return self.repo.url_for_commit(self)
+
+    def log_iter(self, skip, count):
+        for oids in utils.chunked_iter(commitlog(self._id), QSIZE):
+            oids = list(oids)
+            commits = dict(
+                (ci._id, ci) for ci in self.query.find(dict(
+                        _id={'$in': oids})))
+            for oid in oids:
+                ci = commits[oid]
+                ci.set_context(self.repo)
+                yield ci
+
+    def log(self, skip, count):
+        return list(self.log_iter(skip, count))
+
+    def count_revisions(self):
+        result = 0
+        for oid in commitlog(self): result += 1
+        return result
+
+    def context(self):
+        result = dict(prev=None, next=None)
+        if self.parent_ids:
+            result['prev'] = self.query.get(_id=self.parent_ids[0])
+        if self.child_ids:
+            result['next'] = self.query.get(_id=self.child_ids[0])
+        return result
+
+class Tree(RepoObject):
+    # Ephemeral attrs
+    repo=None
+    commit=None
+    parent=None
+    name=None
+
+    def compute_hash(self):
+        '''Compute a hash based on the contents of the tree.  Note that this
+        hash does not necessarily correspond to any actual DVCS hash.
+        '''
+        lines = (
+            [ 'tree' + x.name + x.id for x in self.tree_ids ]
+            + [ 'blob' + x.name + x.id for x in self.blob_ids ]
+            + [ x.type + x.name + x.id for x in self.other_ids ])
+        sha_obj = sha1()
+        for line in sorted(lines):
+            sha_obj.update(line)
+        return sha_obj.hexdigest()
+
+    def set_context(self, commit_or_tree, name=None):
+        assert commit_or_tree is not self
+        self.repo = commit_or_tree.repo
+        if name:
+            self.commit = commit_or_tree.commit
+            self.parent = commit_or_tree
+            self.name = name
+        else:
+            self.commit = commit_or_tree
+
+    def readme(self):
+        name = None
+        text = ''
+        for x in self.blob_ids:
+            if README_RE.match(x.name):
+                name = x.name
+                text = h.really_unicode(self.repo.blob_text(x.id))
+                break
+        if text == '':
+            text = '<p><em>Empty File</em></p>'
+        else:
+            renderer = g.pypeline_markup.renderer(name)
+            if renderer[1]:
+                text = g.pypeline_markup.render(name,text)
+            else:
+                text = '<pre>%s</pre>' % text
+        return (name, text)
+
+    def ls(self):
+        # Load last commit info
+        oids = [ x.id for x in chain(self.tree_ids, self.object_ids, self.other_ids) ]
+        lc_index = dict(
+            (lc.object_id, lc.commit)
+            for lc in LastCommitDoc.m.find(dict(
+                    repo_id=self.repo._id,
+                    object_id={'$in': oids})))
+        results = []
+        def _get_last_commit(oid):
+            lc = lc_index.get(oid)
+            if lc is None:
+                lc = dict(
+                    author=None,
+                    author_email=None,
+                    author_url=None,
+                    date=None,
+                    id=None,
+                    href=None,
+                    shortlink=None,
+                    summary=None)
+            return lc
+        for x in sorted(self.tree_ids, key=lambda x:x.name):
+            results.append(dict(
+                    kind='DIR',
+                    name=x.name,
+                    href=x.name + '/',
+                    last_commit=_get_last_commit(x.id)))
+        for x in sorted(self.blob_ids, key=lambda x:x.name):
+            results.append(dict(
+                    kind='FILE',
+                    name=x.name,
+                    href=x.name + '/',
+                    last_commit=_get_last_commit(x.id)))
+        for x in sorted(self.other_ids, key=lambda x:x.name):
+            results.append(dict(
+                    kind=x.type,
+                    name=x.name,
+                    href=None,
+                    last_commit=_get_last_commit(x.id)))
+        return results
+
+    def path(self):
+        if self.parent:
+            assert self.parent is not self
+            return self.parent.path() + self.name + '/'
+        else:
+            return '/'
+
+    def url(self):
+        return self.commit.url() + 'tree' + self.path()
+
+    @LazyProperty
+    def by_name(self):
+        d = dict((x.name, x) for x in self.other_ids)
+        d.update(
+            (x.name, dict(x, type='tree'))
+            for x in self.tree_ids)
+        d.update(
+            (x.name, dict(x, type='blob'))
+            for x in self.blob_ids)
+        return d
+
+    def is_blob(self, name):
+        return self.by_name[name].type == 'blob'
+
+mapper(Commit, CommitDoc, repository_orm_session)
+mapper(Tree, TreeDoc, repository_orm_session)
+
+def commitlog(commit_id, skip=0, limit=sys.maxint):
+
+    seen = set()
+    def _visit(commit_id):
+        if commit_id in seen: return
+        run = CommitRunDoc.m.get(commit_ids=commit_id)
+        if run is None: return
+        index = False
+        for pos, (oid, time) in enumerate(izip(run.commit_ids, run.commit_times)):
+            if oid == commit_id: index = True
+            elif not index: continue
+            seen.add(oid)
+            ci_times[oid] = time
+            if pos+1 < len(run.commit_ids):
+                ci_parents[oid] = [ run.commit_ids[pos+1] ]
+            else:
+                ci_parents[oid] = run.parent_commit_ids
+        for oid in run.parent_commit_ids:
+            _visit(oid)
+
+    def _gen_ids(commit_id, skip, limit):
+        # Traverse the graph in topo order, yielding commit IDs
+        commits = set([commit_id])
+        new_parent = None
+        while commits and limit:
+            # next commit is latest commit that's valid to log
+            if new_parent in commits:
+                ci = new_parent
+            else:
+                ci = max(commits, key=lambda ci:ci_times[ci])
+            commits.remove(ci)
+            if skip:
+                skip -= 1
+                continue
+            else:
+                limit -= 1
+            yield ci
+            # remove this commit from its parents children and add any childless
+            # parents to the 'ready set'
+            new_parent = None
+            for oid in ci_parents[ci]:
+                children = ci_children[oid]
+                children.discard(ci)
+                if not children:
+                    commits.add(oid)
+                    new_parent = oid
+
+    # Load all the runs to build a commit graph
+    ci_times = {}
+    ci_parents = {}
+    ci_children = defaultdict(set)
+    log.info('Build commit graph')
+    _visit(commit_id)
+    for oid, parents in ci_parents.iteritems():
+        for ci_parent in parents:
+            ci_children[ci_parent].add(oid)
+
+    return _gen_ids(commit_id, skip, limit)

--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -49,6 +49,9 @@ class RepositoryImplementation(object):
     def commit(self, revision): # pragma no cover
         raise NotImplementedError, 'commit'
 
+    def all_commit_ids(self): # pragma no cover
+        raise NotImplementedError, 'all_commit_ids'
+
     def new_commits(self, all_commits=False): # pragma no cover
         '''Return a list of native commits in topological order (heads first).
 
@@ -74,6 +77,10 @@ class RepositoryImplementation(object):
         '''Refresh the data in the commit object 'ci' with data from the repo'''
         raise NotImplementedError, 'refresh_commit'
 
+    def refresh_commit_info(self, oid): # pragma no cover
+        '''Refresh the data in the commit with id oid'''
+        raise NotImplementedError, 'refresh_commit_info'
+
     def refresh_tree(self, tree): # pragma no cover
         '''Refresh the data in the tree object 'tree' with data from the repo'''
         raise NotImplementedError, 'refresh_tree'
@@ -105,8 +112,8 @@ class RepositoryImplementation(object):
         '''Return a file-like object that contains the contents of the blob'''
         raise NotImplementedError, 'open_blob'
 
-    def shorthand_for_commit(self, commit):
-        return '[%s]' % commit.object_id[:6]
+    def shorthand_for_commit(self, oid):
+        return '[%s]' % oid
 
     def symbolics_for_commit(self, commit):
         '''Return symbolic branch and tag names for a commit.
@@ -188,6 +195,10 @@ class Repository(Artifact):
         return self._impl.init()
     def commit(self, rev):
         return self._impl.commit(rev)
+    def all_commit_ids(self):
+        return self._impl.all_commit_ids()
+    def refresh_commit_info(self, oid, seen):
+        return self._impl.refresh_commit_info(oid, seen)
     def commit_context(self, commit):
         return self._impl.commit_context(commit)
     def open_blob(self, blob):

--- a/Allura/test-light.py
+++ b/Allura/test-light.py
@@ -3,60 +3,49 @@ import logging
 from collections import defaultdict
 from itertools import chain, izip
 from datetime import datetime
+from cPickle import dumps
 
+import bson
 from pylons import c
 from pymongo.errors import DuplicateKeyError
 
 from ming.base import Object
 
-from allura import model as M
 from allura.lib import helpers as h
 from allura.lib import utils
+from allura.model.repo import CommitDoc, TreeDoc, TreesDoc, DiffInfoDoc
+from allura.model.repo import LastCommitDoc, CommitRunDoc
+from allura.model.repo import Commit
+from allura.model.index import ArtifactReferenceDoc, ShortlinkDoc
 
 log = logging.getLogger(__name__)
 
 QSIZE=100
 
-def dolog():
-    h.set_context('test', 'code')
-    repo = c.app.repo._impl._git
-    oid = repo.commit(repo.heads[0]).hexsha
-    log.info('start')
-    for i, ci in enumerate(commitlog(oid)):
-        print repr(ci)
-    log.info('done')
-
 def main():
     if len(sys.argv) > 1:
         h.set_context('test')
         c.project.install_app('Git', 'code', 'Code', init_from_url='/home/rick446/src/forge')
     h.set_context('test', 'code')
-    M.repo.Commit.m.remove({})
-    M.repo.Tree.m.remove({})
-    M.repo.Trees.m.remove({})
-    M.repo.DiffInfo.m.remove({})
-    M.repo.LastCommit.m.remove({})
-    M.repo.BasicBlock.m.remove({})
-    repo = c.app.repo._impl._git
-
-    # Get all commits
-    seen = set()
-    all_commit_ids = []
-    for head in repo.heads:
-        for ci in repo.iter_commits(head, topo_order=True):
-            if ci.binsha in seen: continue
-            seen.add(ci.binsha)
-            all_commit_ids.append(ci.hexsha)
-
-    # Skip commits that are already in the DB
+    CommitDoc.m.remove({})
+    TreeDoc.m.remove({})
+    TreesDoc.m.remove({})
+    DiffInfoDoc.m.remove({})
+    LastCommitDoc.m.remove({})
+    CommitRunDoc.m.remove({})
+
+    # Get all commits (repo-specific)
+    all_commit_ids = list(c.app.repo.all_commit_ids())
+
+    # Skip commits that are already in the DB (repo-agnostic)
     commit_ids = unknown_commit_ids(all_commit_ids)
     # commit_ids = commit_ids[:500]
     log.info('Refreshing %d commits', len(commit_ids))
 
-    # Refresh commits
+    # Refresh commits (repo-specific)
+    seen = set()
     for i, oid in enumerate(commit_ids):
-        ci = repo.rev_parse(oid)
-        refresh_commit_info(ci, seen)
+        c.app.repo.refresh_commit_info(oid, seen)
         if (i+1) % 100 == 0:
             log.info('Refresh commit info %d: %s', (i+1), oid)
 
@@ -64,14 +53,14 @@ def main():
     # Everything below here is repo-agnostic
     #############################################
 
-    refresh_repo(commit_ids, c.app.repo._id)
+    refresh_repo(commit_ids, c.app.repo)
 
     # Refresh child references
     seen = set()
     parents = set()
 
     for i, oid in enumerate(commit_ids):
-        ci = M.repo.Commit.m.find(dict(_id=oid), validate=False).next()
+        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
         refresh_children(ci)
         seen.add(ci._id)
         parents.update(ci.parent_ids)
@@ -79,28 +68,22 @@ def main():
             log.info('Refresh child (a) info %d: %s', (i+1), ci._id)
     for j, oid in enumerate(parents-seen):
         try:
-            ci = M.repo.Commit.m.find(dict(_id=oid), validate=False).next()
+            ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
         except StopIteration:
             continue
         refresh_children(ci)
         if (i + j + 1) % 100 == 0:
             log.info('Refresh child (b) info %d: %s', (i + j + 1), ci._id)
 
-    # Refresh basic blocks
-    bbb = BasicBlockBuilder(commit_ids)
-    bbb.run()
-    bbb.cleanup()
-
-    # Verify the log
-    log.info('Logging via basic blocks')
-    for i, ci in enumerate(commitlog(commit_ids[0])):
-        pass
-    log.info('... done (%d commits from %s)', i+1, commit_ids[0])
+    # Refresh commit runs
+    rb = CommitRunBuilder(commit_ids)
+    rb.run()
+    rb.cleanup()
 
     # Refresh trees
     cache = {}
     for i, oid in enumerate(commit_ids):
-        ci = M.repo.Commit.m.find(dict(_id=oid), validate=False).next()
+        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
         cache = refresh_commit_trees(ci, cache)
         if (i+1) % 100 == 0:
             log.info('Refresh commit trees %d: %s', (i+1), ci._id)
@@ -108,13 +91,13 @@ def main():
     # Compute diffs
     cache = {}
     for i, oid in enumerate(commit_ids):
-        ci = M.repo.Commit.m.find(dict(_id=oid), validate=False).next()
+        ci = CommitDoc.m.find(dict(_id=oid), validate=False).next()
         compute_diffs(c.app.repo._id, cache, ci)
         if (i+1) % 100 == 0:
             log.info('Compute diffs %d: %s', (i+1), ci._id)
 
 def refresh_commit_trees(ci, cache):
-    trees_doc = M.repo.Trees(dict(
+    trees_doc = TreesDoc(dict(
             _id=ci._id,
             tree_ids = list(trees(ci.tree_id, cache))))
     trees_doc.m.save(safe=False)
@@ -124,10 +107,10 @@ def refresh_commit_trees(ci, cache):
     return new_cache
 
 def refresh_commit_info(ci, seen):
-    if M.repo.Commit.m.find(dict(_id=ci.hexsha)).count() != 0:
+    if CommitDoc.m.find(dict(_id=ci.hexsha)).count() != 0:
         return False
     try:
-        ci_doc = M.repo.Commit(dict(
+        ci_doc = CommitDoc(dict(
                 _id=ci.hexsha,
                 tree_id=ci.tree.hexsha,
                 committed = Object(
@@ -149,109 +132,128 @@ def refresh_commit_info(ci, seen):
     refresh_tree(ci.tree, seen)
     return True
 
-def refresh_repo(commit_ids, repo_id):
+def refresh_repo(commit_ids, repo):
     for oids in utils.chunked_iter(commit_ids, QSIZE):
         oids = list(oids)
-        M.repo.Commit.m.update_partial(
+        # Create shortlinks and artifactrefs
+        for oid in oids:
+            index_id = 'allura.model.repo.Commit#' + oid
+            ref = ArtifactReferenceDoc(dict(
+                    _id=index_id,
+                    artifact_reference=dict(
+                        cls=dumps(Commit),
+                        project_id=repo.app.config.project_id,
+                    app_config_id=repo.app.config._id,
+                        artifact_id=oid),
+                    references=[]))
+            link = ShortlinkDoc(dict(
+                    _id=bson.ObjectId(),
+                    ref_id=index_id,
+                    project_id=repo.app.config.project_id,
+                    app_config_id=repo.app.config._id,
+                    link=repo.shorthand_for_commit(oid),
+                    url=repo.url() + 'ci/' + oid + '/'))
+            ref.m.save(safe=False, validate=False)
+            link.m.save(safe=False, validate=False)
+        CommitDoc.m.update_partial(
             dict(
                 _id={'$in': oids},
-                repo_ids={'$ne': repo_id}),
-            {'$addToSet': dict(repo_ids=repo_id)},
+                repo_ids={'$ne': repo._id}),
+            {'$addToSet': dict(repo_ids=repo._id)},
             multi=True)
 
 def refresh_children(ci):
-    M.repo.Commit.m.update_partial(
+    CommitDoc.m.update_partial(
         dict(_id={'$in': ci.parent_ids}),
         {'$addToSet': dict(child_ids=ci._id)},
         multi=True)
 
-class BasicBlockBuilder(object):
+class CommitRunBuilder(object):
 
     def __init__(self, commit_ids):
         self.commit_ids = commit_ids
-        self.block_index = {} # by commit ID
-        self.blocks = {}          # by block ID
-        self.reasons = {}        # reasons to stop merging blocks
+        self.run_index = {} # by commit ID
+        self.runs = {}          # by run ID
+        self.reasons = {}    # reasons to stop merging runs
 
     def run(self):
         for oids in utils.chunked_iter(self.commit_ids, QSIZE):
             oids = list(oids)
-            commits = list(M.repo.Commit.m.find(dict(_id={'$in':oids})))
+            commits = list(CommitDoc.m.find(dict(_id={'$in':oids})))
             for ci in commits:
-                if ci._id in self.block_index: continue
-                self.block_index[ci._id] = ci._id
-                self.blocks[ci._id] = M.repo.BasicBlock(dict(
+                if ci._id in self.run_index: continue
+                self.run_index[ci._id] = ci._id
+                self.runs[ci._id] = CommitRunDoc(dict(
                         _id=ci._id,
                         parent_commit_ids=ci.parent_ids,
                         commit_ids=[ci._id],
                         commit_times=[ci.authored.date]))
-            self.merge_blocks()
-        log.info('%d basic blocks', len(self.blocks))
-        for bid, bb in sorted(self.blocks.items()):
-            log.info('%32s: %r', self.reasons.get(bid, 'none'), bb)
-        for bb in self.blocks.itervalues():
-            bb.m.save()
-        return self.blocks
-
-    def _all_blocks(self):
-        blocks = {}
+            self.merge_runs()
+        log.info('%d runs', len(self.runs))
+        for rid, run in sorted(self.runs.items()):
+            log.info('%32s: %r', self.reasons.get(rid, 'none'), run._id)
+        for run in self.runs.itervalues():
+            run.m.save()
+        return self.runs
+
+    def _all_runs(self):
+        runs = {}
         for oids in utils.chunked_iter(self.commit_ids, QSIZE):
             oids = list(oids)
-            for bb in M.repo.BasicBlock.m.find(dict(commit_ids={'$in': oids})):
-                blocks[bb._id] = bb
-        seen_bids = set()
-        blocks = blocks.values()
-        while blocks:
-            bb = blocks.pop()
-            if bb._id in seen_bids: continue
-            seen_bids.add(bb._id)
-            yield bb
-            for bb in M.repo.BasicBlock.m.find(
-                dict(commit_ids={'$in':bb.parent_commit_ids})):
-                blocks.append(bb)
+            for run in CommitRunDoc.m.find(dict(commit_ids={'$in': oids})):
+                runs[run._id] = run
+        seen_run_ids = set()
+        runs = runs.values()
+        while runs:
+            run = runs.pop()
+            if run._id in seen_run_ids: continue
+            seen_run_ids.add(run._id)
+            yield run
+            for run in CommitRunDoc.m.find(
+                dict(commit_ids={'$in':run.parent_commit_ids})):
+                runs.append(run)
 
     def cleanup(self):
-        '''Delete non-maximal basic blocks'''
-        for bb1 in self._all_blocks():
-            for bb2 in M.repo.BasicBlock.m.find(dict(
-                    commit_ids=bb1.commit_ids[0])):
-                if bb2._id == bb1._id: continue
-                log.info('... delete %r (part of %r)', bb2, bb1)
-                import pdb; pdb.set_trace()
-                bb2.m.delete()
-
-    def merge_blocks(self):
+        '''Delete non-maximal runs'''
+        for run1 in self._all_runs():
+            for run2 in CommitRunDoc.m.find(dict(
+                    commit_ids=run1.commit_ids[0])):
+                if run1._id == run2._id: continue
+                log.info('... delete %r (part of %r)', run2, run1)
+                run2.m.delete()
+
+    def merge_runs(self):
         while True:
-            for bid, bb in self.blocks.iteritems():
-                if len(bb.parent_commit_ids) != 1:
-                    self.reasons[bid] = '%d parents' % len(bb.parent_commit_ids)
+            for run_id, run in self.runs.iteritems():
+                if len(run.parent_commit_ids) != 1:
+                    self.reasons[run_id] = '%d parents' % len(run.parent_commit_ids)
                     continue
-                p_oid = bb.parent_commit_ids[0]
-                p_bid = self.block_index.get(p_oid)
-                if p_bid is None:
-                    self.reasons[bid] = 'parent commit not found'
+                p_oid = run.parent_commit_ids[0]
+                p_run_id = self.run_index.get(p_oid)
+                if p_run_id is None:
+                    self.reasons[run_id] = 'parent commit not found'
                     continue
-                p_bb = self.blocks.get(p_bid)
-                if p_bb is None:
-                    self.reasons[bid] = 'parent block not found'
+                p_run = self.runs.get(p_run_id)
+                if p_run is None:
+                    self.reasons[run_id] = 'parent run not found'
                     continue
-                if p_bb.commit_ids[0] != p_oid:
-                    self.reasons[bid] = 'parent does not start with parent commit'
+                if p_run.commit_ids[0] != p_oid:
+                    self.reasons[run_id] = 'parent does not start with parent commit'
                     continue
-                bb.commit_ids += p_bb.commit_ids
-                bb.commit_times += p_bb.commit_times
-                bb.parent_commit_ids = p_bb.parent_commit_ids
-                for oid in p_bb.commit_ids:
-                    self.block_index[oid] = bid
+                run.commit_ids += p_run.commit_ids
+                run.commit_times += p_run.commit_times
+                run.parent_commit_ids = p_run.parent_commit_ids
+                for oid in p_run.commit_ids:
+                    self.run_index[oid] = run_id
                 break
             else:
                 break
-            del self.blocks[p_bid]
+            del self.runs[p_run_id]
 
 def refresh_tree(t, seen):
     if t.binsha in seen: return
     seen.add(t.binsha)
-    doc = M.repo.Tree(dict(
+    doc = TreeDoc(dict(
             _id=t.hexsha,
             tree_ids=[],
             blob_ids=[],
@@ -274,7 +276,7 @@ def trees(id, cache):
     yield id
     entries = cache.get(id, None)
     if entries is None:
-        t = M.repo.Tree.m.get(_id=id)
+        t = TreeDoc.m.get(_id=id)
         entries = [ o.id for o in t.tree_ids ]
         cache[id] = entries
     for i in entries:
@@ -284,7 +286,7 @@ def trees(id, cache):
 def unknown_commit_ids(all_commit_ids):
     result = []
     for chunk in utils.chunked_iter(all_commit_ids, QSIZE):
-        q = M.repo.Commit.m.find(_id={'$in':chunk})
+        q = CommitDoc.m.find(_id={'$in':chunk})
         known_commit_ids = set(ci._id for ci in q)
         result += [ oid for oid in chunk if oid not in known_commit_ids ]
     return result
@@ -298,20 +300,20 @@ def compute_diffs(repo_id, tree_cache, rhs_ci):
             for xx in _walk_tree(tree_index[x.id], tree_index):
                 yield xx
 
-    rhs_tree_ids = M.repo.Trees.m.get(_id=rhs_ci._id).tree_ids
+    rhs_tree_ids = TreesDoc.m.get(_id=rhs_ci._id).tree_ids
     if rhs_ci.parent_ids:
-        lhs_ci = M.repo.Commit.m.get(_id=rhs_ci.parent_ids[0])
+        lhs_ci = CommitDoc.m.get(_id=rhs_ci.parent_ids[0])
     else:
         lhs_ci = None
     if lhs_ci is not None:
-        lhs_tree_ids = M.repo.Trees.m.get(_id=lhs_ci._id).tree_ids
+        lhs_tree_ids = TreesDoc.m.get(_id=lhs_ci._id).tree_ids
     else:
         lhs_tree_ids = []
     new_tree_ids = [
         tid for tid in chain(lhs_tree_ids, rhs_tree_ids)
         if tid not in tree_cache ]
     tree_index = dict(
-        (t._id, t) for t in M.repo.Tree.m.find(dict(_id={'$in': new_tree_ids}),validate=False))
+        (t._id, t) for t in TreeDoc.m.find(dict(_id={'$in': new_tree_ids}),validate=False))
     tree_index.update(tree_cache)
     rhs_tree_ids_set = set(rhs_tree_ids)
     tree_cache.clear()
@@ -328,83 +330,17 @@ def compute_diffs(repo_id, tree_cache, rhs_ci):
             dict(name=name, lhs_id=lhs_id, rhs_id=rhs_id))
         # Set last commit info
         if rhs_id is not None:
-            M.repo.LastCommit.set_last_commit(repo_id, rhs_id, rhs_ci)
+            _set_last_commit(repo_id, rhs_id, rhs_ci)
         rhs_tree = tree_index.get(rhs_id, None)
         if rhs_tree is not None:
             for oid in _walk_tree(rhs_tree, tree_index):
-                M.repo.LastCommit.set_last_commit(repo_id, oid, rhs_ci)
-    di = M.repo.DiffInfo(dict(
+                _set_last_commit(repo_id, oid, rhs_ci)
+    di = DiffInfoDoc(dict(
             _id=rhs_ci._id,
             differences=differences))
     di.m.save()
     return tree_cache
 
-def commitlog(commit_id, skip=0, limit=sys.maxint):
-
-    seen = set()
-    def _visit(commit_id):
-        if commit_id in seen: return
-        bb = M.repo.BasicBlock.m.get(commit_ids=commit_id)
-        if bb is None: return
-        index = False
-        for pos, (oid, time) in enumerate(izip(bb.commit_ids, bb.commit_times)):
-            if oid == commit_id: index = True
-            elif not index: continue
-            seen.add(oid)
-            ci_times[oid] = time
-            if pos+1 < len(bb.commit_ids):
-                ci_parents[oid] = [ bb.commit_ids[pos+1] ]
-            else:
-                ci_parents[oid] = bb.parent_commit_ids
-        for oid in bb.parent_commit_ids:
-            _visit(oid)
-
-    def _gen_ids(commit_id, skip, limit):
-        # Traverse the graph in topo order, yielding commit IDs
-        commits = set([commit_id])
-        new_parent = None
-        while commits and limit:
-            # next commit is latest commit that's valid to log
-            if new_parent in commits:
-                ci = new_parent
-            else:
-                ci = max(commits, key=lambda ci:ci_times[ci])
-            commits.remove(ci)
-            if skip:
-                skip -= 1
-                continue
-            else:
-                limit -= 1
-            yield ci
-            # remove this commit from its parents children and add any childless
-            # parents to the 'ready set'
-            new_parent = None
-            for oid in ci_parents[ci]:
-                children = ci_children[oid]
-                children.discard(ci)
-                if not children:
-                    commits.add(oid)
-                    new_parent = oid
-
-    # Load all the blocks to build a commit graph
-    ci_times = {}
-    ci_parents = {}
-    ci_children = defaultdict(set)
-    log.info('Build commit graph')
-    _visit(commit_id)
-    for oid, parents in ci_parents.iteritems():
-        for ci_parent in parents:
-            ci_children[ci_parent].add(oid)
-
-    # Convert oids to commit objects
-    log.info('Traverse commit graph')
-    for oids in utils.chunked_iter(_gen_ids(commit_id, skip, limit), QSIZE):
-        oids = list(oids)
-        index = dict(
-            (ci._id, ci) for ci in M.repo.Commit.m.find(dict(_id={'$in': oids})))
-        for oid in oids:
-            yield index[oid]
-
 def _diff_trees(lhs, rhs, index, *path):
     def _fq(name):
         return '/'.join(reversed(
@@ -441,6 +377,24 @@ def _diff_trees(lhs, rhs, index, *path):
     for name, id in rhs_blob_ids.items():
         yield (_fq(name), None, id)
 
+def _set_last_commit(repo_id, oid, commit):
+    lc = LastCommitDoc(dict(
+            _id='%s:%s' % (repo_id, oid),
+            repo_id=repo_id,
+            object_id=oid,
+            commit_info=dict(
+                id=commit._id,
+                author=commit.authored.name,
+                author_email=commit.authored.email,
+                date=commit.authored.date,
+                # author_url=commit.author_url,
+                # href=commit.url(),
+                # shortlink=commit.shorthand_id(),
+                # summary=commit.summary
+                )))
+    lc.m.save(safe=False)
+    return lc
+
 if __name__ == '__main__':
     main()
     # dolog()

--- a/ForgeGit/forgegit/model/git_repo.py
+++ b/ForgeGit/forgegit/model/git_repo.py
@@ -117,6 +117,14 @@ class GitImplementation(M.RepositoryImplementation):
         result.set_context(self._repo)
         return result
 
+    def all_commit_ids(self):
+        seen = set()
+        for head in self._git.heads:
+            for ci in self._git.iter_commits(head, topo_order=True):
+                if ci.binsha in seen: continue
+                seen.add(ci.binsha)
+                yield ci.hexsha
+
     def new_commits(self, all_commits=False):
         commits = list(self._git.iter_commits(topo_order=True))
         if all_commits: return commits
@@ -184,6 +192,57 @@ class GitImplementation(M.RepositoryImplementation):
         self._build_manifest(native_ci)
         self.refresh_tree(root_entry)
 
+    def refresh_commit_info(self, oid, seen):
+        from allura.model.repo import CommitDoc
+        if CommitDoc.m.find(dict(_id=oid)).count():
+            return False
+        try:
+            ci = self._git.rev_parse(oid)
+            ci_doc = CommitDoc(dict(
+                    _id=ci.hexsha,
+                    tree_id=ci.tree.hexsha,
+                    committed = Object(
+                        name=h.really_unicode(ci.committer.name),
+                        email=h.really_unicode(ci.committer.email),
+                        date=datetime.utcfromtimestamp(
+                            ci.committed_date-ci.committer_tz_offset)),
+                    authored = Object(
+                        name=h.really_unicode(ci.author.name),
+                        email=h.really_unicode(ci.author.email),
+                        date=datetime.utcfromtimestamp(
+                            ci.authored_date-ci.author_tz_offset)),
+                    message=h.really_unicode(ci.message or ''),
+                    child_ids=[],
+                    parent_ids = [ p.hexsha for p in ci.parents ]))
+            ci_doc.m.insert(safe=True)
+        except DuplicateKeyError:
+            return False
+        self.refresh_tree_info(ci.tree, seen)
+        return True
+
+    def refresh_tree_info(self, tree, seen):
+        from allura.model.repo import TreeDoc
+        if tree.binsha in seen: return
+        seen.add(tree.binsha)
+        doc = TreeDoc(dict(
+                _id=tree.hexsha,
+                tree_ids=[],
+                blob_ids=[],
+                other_ids=[]))
+        for o in tree:
+            obj = Object(
+                name=h.really_unicode(o.name),
+                id=o.hexsha)
+            if o.type == 'tree':
+                self.refresh_tree_info(o, seen)
+                doc.tree_ids.append(obj)
+            elif o.type == 'blob':
+                doc.blob_ids.append(obj)
+            else:
+                obj.type = o.type
+                doc.other_ids.append(obj)
+        doc.m.save(safe=False)
+
     def _build_manifest(self, native_ci):
         '''Build the manifest for this commit (mapof all paths to trees/blobs)