diff options
| -rwxr-xr-x | git-remote-gittorrent | 162 | ||||
| -rwxr-xr-x | gittorrentd | 67 |
2 files changed, 157 insertions, 72 deletions
diff --git a/git-remote-gittorrent b/git-remote-gittorrent index 49c8363..5c9f038 100755 --- a/git-remote-gittorrent +++ b/git-remote-gittorrent @@ -3,7 +3,6 @@ var Chalk = require('chalk') var DHT = require('bittorrent-dht') var exec = require('child_process').exec -var fs = require('fs') var hat = require('hat') var magnet = require('magnet-uri') var prettyjson = require('prettyjson') @@ -34,11 +33,57 @@ var dht = new DHT({ bootstrap: bootstrap }) +// After building a dictionary of references (sha's to branch names), responds +// to git's "list" and "fetch" commands. +function talk_to_git (refs) { + process.stdin.setEncoding('utf8') + var didFetch = false + process.stdin.on('readable', function () { + var chunk = process.stdin.read() + if (chunk === 'capabilities\n') { + process.stdout.write('fetch\n\n') + } else if (chunk === 'list\n') { + Object.keys(refs).forEach(function (branch, i) { + process.stdout.write(refs[branch] + ' ' + branch + '\n') + }) + process.stdout.write('\n') + } else if (chunk && chunk.search(/^fetch/) !== -1) { + didFetch = true + chunk.split(/\n/).forEach(function (line) { + if (line === '') { + return + } + // Format: "fetch sha branch" + line = line.split(/\s/) + get_infohash(line[1], line[2]) + }) + } else if (chunk && chunk !== '' && chunk !== '\n') { + console.warn('unhandled command: "' + chunk + '"') + } + if (chunk === '\n') { + process.stdout.write('\n') + if (!didFetch) { + // If git already has all the refs it needs, we should exit now. + process.exit() + } + return + } + }) + process.stdout.on('error', function () { + // stdout was closed + }) +} + +var remotename = process.argv[2] var url = process.argv[3] var matches = url.match(/gittorrent:\/\/([a-f0-9]{40})\/(.*)/) +var refs = {} // Maps branch names to sha's. if (matches) { var key = matches[1] var reponame = matches[2] + if (remotename.search(/^gittorrent:\/\//) !== -1) { + remotename = key + } dht.on('ready', function () { var val = new Buffer(key, 'hex') dht.get(val, function (err, res) { @@ -47,65 +92,89 @@ if (matches) { } var json = res.v.toString() var repos = JSON.parse(json) - console.warn('\nMutable key ' + chalk.green(key) + ' returned:\n' + prettyjson.render(repos, {keysColor: 'yellow'})) - get_infohash(repos.repositories[reponame].master) + console.warn('\nMutable key ' + chalk.green(key) + ' returned:\n' + + prettyjson.render(repos, {keysColor: 'yellow', valuesColor: 'green'})) + talk_to_git(repos.repositories[reponame]) }) }) } else { url = url.replace(/^gittorrent:/i, 'git:') - exec('git ls-remote ' + url + ' HEAD', function (err, stdout, stderr) { + exec('git ls-remote ' + url, function (err, stdout, stderr) { if (err !== null) { die(err) } var lines = stdout.split('\n') - if (lines.length !== 2) { + if (lines.length < 2) { die("Didn't get back a single HEAD ref: " + lines) } - var line = lines[0].split('\t') - var ref = line[0] - var head = line[1] - if (head !== 'HEAD') { - die("Couldn't parse the ref line: " + ref, head) - } - if (ref.length !== 40) { - die('Was expecting a 40-byte sha: ' + ref) - } - dht.on('ready', function () { - get_infohash(ref) + lines.forEach(function (line) { + if (line === '') { + // Last line: publish + dht.on('ready', function () { + talk_to_git(refs) + }) + return + } + + line = line.split('\t') + var sha = line[0] + var branch = line[1] + if (sha.length !== 40) { + console.warn('Was expecting a 40-byte sha: ' + sha + '\n') + console.warn('on line: ' + line.join('\t')) + } + refs[branch] = sha }) }) + } -function get_infohash (ref) { - // We use console.warn (stderr) because git ignores our writes to stdout. - console.warn('\nOkay, we want to get: ' + chalk.green(ref) + '\n') +var fetching = {} // Maps shas -> {got: <bool>, swarm, branches: [...]} +var todo = 0 // The number of sha's we have yet to fetch. We will not exit + // until this equals zero. +dht.on('peer', function (addr, hash, from) { + var goal = fetching[hash] + if (!goal.peer) { + todo++ + goal.peer = true + } + goal.swarm.addPeer(addr) +}) - process.stdin.setEncoding('utf8') - process.stdin.on('readable', function () { - var chunk = process.stdin.read() - if (chunk === 'capabilities\n') { - process.stdout.write('fetch\n\n') - } - if (chunk === 'list\n') { - process.stdout.write(ref + ' refs/heads/master\n\n') - } - }) - process.stdout.on('error', function () { - // stdout was closed +function update_ref (sha) { + fetching[sha].branches.forEach(function (branch) { + branch = remotename + '/' + branch + spawn('git', ['update-ref', branch, sha]) + console.warn('git update-ref ' + chalk.yellow(branch) + ' ' + + chalk.green(sha)) }) +} + +function get_infohash (sha, branch) { + branch = branch.replace(/^refs\/(heads\/)?/, '') + branch = branch.replace(/\/head$/, '') + + // We use console.warn (stderr) because git ignores our writes to stdout. + console.warn('\nOkay, we want to get ' + chalk.yellow(branch) + ': ' + + chalk.green(sha) + '\n') - var magnetUri = 'magnet:?xt=urn:btih:' + ref + if (sha in fetching) { + fetching[sha].branches.push(branch) + return // Prevent starting a redundant lookups + } + + var info = {got: false, peer: false, swarm: null, branches: [branch]} + fetching[sha] = info + + var magnetUri = 'magnet:?xt=urn:btih:' + sha var parsed = magnet(magnetUri) dht.lookup(parsed.infoHash) - dht.on('peer', function (addr, hash, from) { - swarm.addPeer(addr) - }) - var peerId = new Buffer('-WW' + VERSION + '-' + hat(48), 'utf8') - var swarm = new Swarm(parsed.infoHash, peerId) - swarm.on('wire', function (wire, addr) { - console.warn('Adding swarm peer: ' + chalk.green(addr) + '\n') + info.swarm = new Swarm(parsed.infoHash, peerId) + info.swarm.on('wire', function (wire, addr) { + console.warn('Adding swarm peer: ' + chalk.green(addr) + ' for ' + + chalk.red(parsed.infoHash) + '\n') wire.use(ut_gittorrent()) wire.ut_gittorrent.on('handshake', function () { wire.ut_gittorrent.ask(parsed.infoHash) @@ -120,20 +189,21 @@ function get_infohash (ref) { client.download(infoHash, function (torrent) { console.warn('Downloading git pack with infohash: ' + chalk.green(infoHash) + '\n') torrent.on('done', function (done) { + fetching[sha].got = true + var stream = torrent.files[0].createReadStream() var unpack = spawn('git', ['index-pack', '--stdin', '-v', '--fix-thin']) stream.pipe(unpack.stdin) unpack.stderr.pipe(process.stderr) unpack.on('exit', function (code) { - var targetdir = process.env['GIT_DIR'] - var stream = fs.createWriteStream(targetdir + '/refs/heads/master') - stream.once('open', function (fd) { - stream.write(ref + '\n') - stream.end() - // These writes are actually necessary for git to finish checkout. + update_ref(sha) + todo-- + if (todo <= 0) { + // These writes are actually necessary for git to finish + // checkout. process.stdout.write('\n\n') process.exit() - }) + } }) }) }) diff --git a/gittorrentd b/gittorrentd index 130731f..52e59fa 100755 --- a/gittorrentd +++ b/gittorrentd @@ -36,7 +36,6 @@ var dht = new DHT({ dht.listen(6881) var announcedRefs = { - master: {} } var userProfile = { repositories: {} @@ -82,37 +81,52 @@ dht.on('ready', function () { console.log('in repo ' + repo) repo = repo.replace(/git-daemon-export-ok$/, '') console.log(repo) + + var reponame = repo.replace(/\/.git\/$/, '') + userProfile.repositories[reponame] = {} + var upload = spawn('git-upload-pack', ['--strict', repo]) upload.stdout.on('data', function (line) { var lines = line.toString().split('\n') lines.forEach(function (line) { var arr = line.toString().split(' ') - if (arr.length === 2) { - var sha = arr[0].toString() - // First four chars are git-upload-pack's length-of-line metadata. - sha = sha.substring(4) + if (arr.length < 2) { + return + } + var sha = arr[0].toString() + // First four chars are git-upload-pack's length-of-line metadata. + sha = sha.substring(4) + if (arr.length == 2) { var ref = arr[1].toString() - if (ref.search(/^refs\/heads\//) !== -1 || ref.search(/^refs\/remotes\//) !== -1) { - if (!announcedRefs.master[sha]) { - console.log('Announcing ' + sha + ' for ref ' + ref + ' on repo ' + repo) - announcedRefs.master[sha] = repo - var reponame = repo.replace(/\/.git\/$/, '') - userProfile.repositories[reponame] = {} - userProfile.repositories[reponame].master = sha - // Callback counting for repos - count-- - if (count <= 0) { - publish_mutable_key() + var branch = ref.match(/^refs\/heads\/(.*)/) + // FIXME: Can't pull in too many branches. + // if (!branch) { + // branch = ref.match(/^refs\/remotes\/(.*)/) + // } + if (branch) { + userProfile.repositories[reponame][ref] = sha + } + if (branch && !announcedRefs[sha]) { + branch = branch[1] + console.log('Announcing ' + sha + ' for ' + branch + ' on repo ' + repo) + announcedRefs[sha] = repo + dht.announce(sha, 30000, function (err) { + if (err !== null) { + console.log('Announced ' + sha) } - dht.announce(sha, 30000, function (err) { - if (err !== null) { - console.log('Announced ' + sha) - } - }) - } + }) } + } else if (arr.length > 2 && arr[1].search(/^HEAD/) !== -1) { + // Probably the first line; line[0] has the hash, line[1] has HEAD, + // and beyond are the supported features. + userProfile.repositories[reponame]['HEAD'] = sha } }) + // Callback counting for repos + count-- + if (count <= 0) { + publish_mutable_key() + } }) upload.stdout.on('end', function () { console.log('end') @@ -154,7 +168,7 @@ dht.on('ready', function () { wire.use(ut_metadata()) socket.pipe(wire).pipe(socket) wire.on('handshake', function (infoHash, peerId) { - console.log('Received handshake for ' + infoHash) + console.log('Received handshake for ' + infoHash.toString('hex')) var myPeerId = new Buffer('-WW' + VERSION + '-' + hat(48), 'utf8') wire.handshake(new Buffer(infoHash), new Buffer(myPeerId)) }) @@ -162,10 +176,11 @@ dht.on('ready', function () { console.error('calling git pack-objects') var filename = sha + '.pack' var stream = fs.createWriteStream(filename) - if (!announcedRefs.master[sha]) { - console.error('Asked for an unknown sha!') + if (!announcedRefs[sha]) { + console.error('Asked for an unknown sha: ' + sha) + return } - var directory = announcedRefs.master[sha] + var directory = announcedRefs[sha] var pack = spawn('git', ['pack-objects', '--revs', '--thin', '--stdout', '--delta-base-offset'], {cwd: directory}) pack.on('close', function (code) { if (code !== 0) { |