Merge pull request #34 from splinterofchaos/branching

support branches

2 files changed, 157 insertions, 72 deletions

diff --git a/git-remote-gittorrent b/git-remote-gittorrent
index 49c8363..5c9f038 100755
--- a/git-remote-gittorrent
+++ b/git-remote-gittorrent
@@ -3,7 +3,6 @@
 var Chalk = require('chalk')
 var DHT = require('bittorrent-dht')
 var exec = require('child_process').exec
-var fs = require('fs')
 var hat = require('hat')
 var magnet = require('magnet-uri')
 var prettyjson = require('prettyjson')
@@ -34,11 +33,57 @@ var dht = new DHT({
   bootstrap: bootstrap
 })
 
+// After building a dictionary of references (sha's to branch names), responds
+// to git's "list" and "fetch" commands.
+function talk_to_git (refs) {
+  process.stdin.setEncoding('utf8')
+  var didFetch = false
+  process.stdin.on('readable', function () {
+    var chunk = process.stdin.read()
+    if (chunk === 'capabilities\n') {
+      process.stdout.write('fetch\n\n')
+    } else if (chunk === 'list\n') {
+      Object.keys(refs).forEach(function (branch, i) {
+        process.stdout.write(refs[branch] + ' ' + branch + '\n')
+      })
+      process.stdout.write('\n')
+    } else if (chunk && chunk.search(/^fetch/) !== -1) {
+      didFetch = true
+      chunk.split(/\n/).forEach(function (line) {
+        if (line === '') {
+          return
+        }
+        // Format: "fetch sha branch"
+        line = line.split(/\s/)
+        get_infohash(line[1], line[2])
+      })
+    } else if (chunk && chunk !== '' && chunk !== '\n') {
+      console.warn('unhandled command: "' + chunk + '"')
+    }
+    if (chunk === '\n') {
+      process.stdout.write('\n')
+      if (!didFetch) {
+        // If git already has all the refs it needs, we should exit now.
+        process.exit()
+      }
+      return
+    }
+  })
+  process.stdout.on('error', function () {
+    // stdout was closed
+  })
+}
+
+var remotename = process.argv[2]
 var url = process.argv[3]
 var matches = url.match(/gittorrent:\/\/([a-f0-9]{40})\/(.*)/)
+var refs = {}  // Maps branch names to sha's.
 if (matches) {
   var key = matches[1]
   var reponame = matches[2]
+  if (remotename.search(/^gittorrent:\/\//) !== -1) {
+    remotename = key
+  }
   dht.on('ready', function () {
     var val = new Buffer(key, 'hex')
     dht.get(val, function (err, res) {
@@ -47,65 +92,89 @@ if (matches) {
       }
       var json = res.v.toString()
       var repos = JSON.parse(json)
-      console.warn('\nMutable key ' + chalk.green(key) + ' returned:\n' + prettyjson.render(repos, {keysColor: 'yellow'}))
-      get_infohash(repos.repositories[reponame].master)
+      console.warn('\nMutable key ' + chalk.green(key) + ' returned:\n' +
+                   prettyjson.render(repos, {keysColor: 'yellow', valuesColor: 'green'}))
+      talk_to_git(repos.repositories[reponame])
     })
   })
 } else {
   url = url.replace(/^gittorrent:/i, 'git:')
-  exec('git ls-remote ' + url + ' HEAD', function (err, stdout, stderr) {
+  exec('git ls-remote ' + url, function (err, stdout, stderr) {
     if (err !== null) {
       die(err)
     }
     var lines = stdout.split('\n')
-    if (lines.length !== 2) {
+    if (lines.length < 2) {
       die("Didn't get back a single HEAD ref: " + lines)
     }
-    var line = lines[0].split('\t')
-    var ref = line[0]
-    var head = line[1]
-    if (head !== 'HEAD') {
-      die("Couldn't parse the ref line: " + ref, head)
-    }
-    if (ref.length !== 40) {
-      die('Was expecting a 40-byte sha: ' + ref)
-    }
-    dht.on('ready', function () {
-      get_infohash(ref)
+    lines.forEach(function (line) {
+      if (line === '') {
+        // Last line: publish
+        dht.on('ready', function () {
+          talk_to_git(refs)
+        })
+        return
+      }
+
+      line = line.split('\t')
+      var sha = line[0]
+      var branch = line[1]
+      if (sha.length !== 40) {
+        console.warn('Was expecting a 40-byte sha: ' + sha + '\n')
+        console.warn('on line: ' + line.join('\t'))
+      }
+      refs[branch] = sha
     })
   })
+
 }
 
-function get_infohash (ref) {
-  // We use console.warn (stderr) because git ignores our writes to stdout.
-  console.warn('\nOkay, we want to get: ' + chalk.green(ref) + '\n')
+var fetching = {}  // Maps shas -> {got: <bool>, swarm, branches: [...]}
+var todo = 0     // The number of sha's we have yet to fetch. We will not exit
+                 // until this equals zero.
+dht.on('peer', function (addr, hash, from) {
+  var goal = fetching[hash]
+  if (!goal.peer) {
+    todo++
+    goal.peer = true
+  }
+  goal.swarm.addPeer(addr)
+})
 
-  process.stdin.setEncoding('utf8')
-  process.stdin.on('readable', function () {
-    var chunk = process.stdin.read()
-    if (chunk === 'capabilities\n') {
-      process.stdout.write('fetch\n\n')
-    }
-    if (chunk === 'list\n') {
-      process.stdout.write(ref + ' refs/heads/master\n\n')
-    }
-  })
-  process.stdout.on('error', function () {
-    // stdout was closed
+function update_ref (sha) {
+  fetching[sha].branches.forEach(function (branch) {
+    branch = remotename + '/' + branch
+    spawn('git', ['update-ref', branch, sha])
+    console.warn('git update-ref ' + chalk.yellow(branch) + ' ' +
+                 chalk.green(sha))
   })
+}
+
+function get_infohash (sha, branch) {
+  branch = branch.replace(/^refs\/(heads\/)?/, '')
+  branch = branch.replace(/\/head$/, '')
+
+  // We use console.warn (stderr) because git ignores our writes to stdout.
+  console.warn('\nOkay, we want to get ' + chalk.yellow(branch) + ': ' +
+               chalk.green(sha) + '\n')
 
-  var magnetUri = 'magnet:?xt=urn:btih:' + ref
+  if (sha in fetching) {
+    fetching[sha].branches.push(branch)
+    return  // Prevent starting a redundant lookups
+  }
+
+  var info = {got: false, peer: false, swarm: null, branches: [branch]}
+  fetching[sha] = info
+
+  var magnetUri = 'magnet:?xt=urn:btih:' + sha
   var parsed = magnet(magnetUri)
   dht.lookup(parsed.infoHash)
 
-  dht.on('peer', function (addr, hash, from) {
-    swarm.addPeer(addr)
-  })
-
   var peerId = new Buffer('-WW' + VERSION + '-' + hat(48), 'utf8')
-  var swarm = new Swarm(parsed.infoHash, peerId)
-  swarm.on('wire', function (wire, addr) {
-    console.warn('Adding swarm peer: ' + chalk.green(addr) + '\n')
+  info.swarm = new Swarm(parsed.infoHash, peerId)
+  info.swarm.on('wire', function (wire, addr) {
+    console.warn('Adding swarm peer: ' + chalk.green(addr) + ' for ' +
+                 chalk.red(parsed.infoHash) + '\n')
     wire.use(ut_gittorrent())
     wire.ut_gittorrent.on('handshake', function () {
       wire.ut_gittorrent.ask(parsed.infoHash)
@@ -120,20 +189,21 @@ function get_infohash (ref) {
       client.download(infoHash, function (torrent) {
         console.warn('Downloading git pack with infohash: ' + chalk.green(infoHash) + '\n')
         torrent.on('done', function (done) {
+          fetching[sha].got = true
+
           var stream = torrent.files[0].createReadStream()
           var unpack = spawn('git', ['index-pack', '--stdin', '-v', '--fix-thin'])
           stream.pipe(unpack.stdin)
           unpack.stderr.pipe(process.stderr)
           unpack.on('exit', function (code) {
-            var targetdir = process.env['GIT_DIR']
-            var stream = fs.createWriteStream(targetdir + '/refs/heads/master')
-            stream.once('open', function (fd) {
-              stream.write(ref + '\n')
-              stream.end()
-              // These writes are actually necessary for git to finish checkout.
+            update_ref(sha)
+            todo--
+            if (todo <= 0) {
+              // These writes are actually necessary for git to finish
+              // checkout.
               process.stdout.write('\n\n')
               process.exit()
-            })
+            }
           })
         })
       })
diff --git a/gittorrentd b/gittorrentd
index 130731f..52e59fa 100755
--- a/gittorrentd
+++ b/gittorrentd
@@ -36,7 +36,6 @@ var dht = new DHT({
 dht.listen(6881)
 
 var announcedRefs = {
-  master: {}
 }
 var userProfile = {
   repositories: {}
@@ -82,37 +81,52 @@ dht.on('ready', function () {
     console.log('in repo ' + repo)
     repo = repo.replace(/git-daemon-export-ok$/, '')
     console.log(repo)
+
+    var reponame = repo.replace(/\/.git\/$/, '')
+    userProfile.repositories[reponame] = {}
+
     var upload = spawn('git-upload-pack', ['--strict', repo])
     upload.stdout.on('data', function (line) {
       var lines = line.toString().split('\n')
       lines.forEach(function (line) {
         var arr = line.toString().split(' ')
-        if (arr.length === 2) {
-          var sha = arr[0].toString()
-          // First four chars are git-upload-pack's length-of-line metadata.
-          sha = sha.substring(4)
+        if (arr.length < 2) {
+          return
+        }
+        var sha = arr[0].toString()
+        // First four chars are git-upload-pack's length-of-line metadata.
+        sha = sha.substring(4)
+        if (arr.length == 2) {
           var ref = arr[1].toString()
-          if (ref.search(/^refs\/heads\//) !== -1 || ref.search(/^refs\/remotes\//) !== -1) {
-            if (!announcedRefs.master[sha]) {
-              console.log('Announcing ' + sha + ' for ref ' + ref + ' on repo ' + repo)
-              announcedRefs.master[sha] = repo
-              var reponame = repo.replace(/\/.git\/$/, '')
-              userProfile.repositories[reponame] = {}
-              userProfile.repositories[reponame].master = sha
-              // Callback counting for repos
-              count--
-              if (count <= 0) {
-                publish_mutable_key()
+          var branch = ref.match(/^refs\/heads\/(.*)/)
+          // FIXME: Can't pull in too many branches.
+          // if (!branch) {
+          //   branch = ref.match(/^refs\/remotes\/(.*)/)
+          // }
+          if (branch) {
+            userProfile.repositories[reponame][ref] = sha
+          }
+          if (branch && !announcedRefs[sha]) {
+            branch = branch[1]
+            console.log('Announcing ' + sha + ' for ' + branch + ' on repo ' + repo)
+            announcedRefs[sha] = repo
+            dht.announce(sha, 30000, function (err) {
+              if (err !== null) {
+                console.log('Announced ' + sha)
               }
-              dht.announce(sha, 30000, function (err) {
-                if (err !== null) {
-                  console.log('Announced ' + sha)
-                }
-              })
-            }
+            })
           }
+        } else if (arr.length > 2 && arr[1].search(/^HEAD/) !== -1) {
+          // Probably the first line; line[0] has the hash, line[1] has HEAD,
+          // and beyond are the supported features.
+          userProfile.repositories[reponame]['HEAD'] = sha
         }
       })
+      // Callback counting for repos
+      count--
+      if (count <= 0) {
+        publish_mutable_key()
+      }
     })
     upload.stdout.on('end', function () {
       console.log('end')
@@ -154,7 +168,7 @@ dht.on('ready', function () {
     wire.use(ut_metadata())
     socket.pipe(wire).pipe(socket)
     wire.on('handshake', function (infoHash, peerId) {
-      console.log('Received handshake for ' + infoHash)
+      console.log('Received handshake for ' + infoHash.toString('hex'))
       var myPeerId = new Buffer('-WW' + VERSION + '-' + hat(48), 'utf8')
       wire.handshake(new Buffer(infoHash), new Buffer(myPeerId))
     })
@@ -162,10 +176,11 @@ dht.on('ready', function () {
       console.error('calling git pack-objects')
       var filename = sha + '.pack'
       var stream = fs.createWriteStream(filename)
-      if (!announcedRefs.master[sha]) {
-        console.error('Asked for an unknown sha!')
+      if (!announcedRefs[sha]) {
+        console.error('Asked for an unknown sha: ' + sha)
+        return
       }
-      var directory = announcedRefs.master[sha]
+      var directory = announcedRefs[sha]
       var pack = spawn('git', ['pack-objects', '--revs', '--thin', '--stdout', '--delta-base-offset'], {cwd: directory})
       pack.on('close', function (code) {
         if (code !== 0) {