diff options
| -rwxr-xr-x | git-remote-gittorrent | 43 | ||||
| -rw-r--r-- | git.js | 129 | ||||
| -rwxr-xr-x | gittorrentd | 102 |
3 files changed, 182 insertions, 92 deletions
diff --git a/git-remote-gittorrent b/git-remote-gittorrent index aeffc90..bb76dc8 100755 --- a/git-remote-gittorrent +++ b/git-remote-gittorrent @@ -12,6 +12,7 @@ var ut_gittorrent = require('ut_gittorrent') var WebTorrent = require('webtorrent') var zeroFill = require('zero-fill') var config = require('./config') +var git = require('./git') // BitTorrent client version string (used in peer ID). // Generated from package.json major and minor version. For example: @@ -99,31 +100,15 @@ if (matches) { }) } else { url = url.replace(/^gittorrent:/i, 'git:') - exec('git ls-remote ' + url, function (err, stdout, stderr) { - if (err !== null) { + var ls = git.ls(url, function (sha, branch) { + refs[branch] = sha + }) + ls.on('exit', function (err) { + if (err) { die(err) } - var lines = stdout.split('\n') - if (lines.length < 2) { - die("Didn't get back a single HEAD ref: " + lines) - } - lines.forEach(function (line) { - if (line === '') { - // Last line: publish - dht.on('ready', function () { - talk_to_git(refs) - }) - return - } - - line = line.split('\t') - var sha = line[0] - var branch = line[1] - if (sha.length !== 40) { - console.warn('Was expecting a 40-byte sha: ' + sha + '\n') - console.warn('on line: ' + line.join('\t')) - } - refs[branch] = sha + dht.on('ready', function () { + talk_to_git(refs) }) }) } @@ -140,13 +125,6 @@ dht.on('peer', function (addr, hash, from) { goal.swarm.addPeer(addr) }) -function update_ref (sha) { - fetching[sha].branches.forEach(function (branch) { - branch = remotename + '/' + branch - spawn('git', ['update-ref', branch, sha]) - }) -} - function get_infohash (sha, branch) { branch = branch.replace(/^refs\/(heads\/)?/, '') branch = branch.replace(/\/head$/, '') @@ -185,8 +163,10 @@ function get_infohash (sha, branch) { tracker: false }) client.download(infoHash, function (torrent) { - console.warn('Downloading git pack with infohash: ' + chalk.green(infoHash) + '\n') + console.warn('Downloading ' + chalk.green(torrent.files[0].path) + + ' with infohash: ' + chalk.green(infoHash) + '\n') torrent.on('done', function (done) { + console.warn('done downloading: ' + chalk.green(torrent.files[0].path)) fetching[sha].got = true var stream = torrent.files[0].createReadStream() @@ -194,7 +174,6 @@ function get_infohash (sha, branch) { stream.pipe(unpack.stdin) unpack.stderr.pipe(process.stderr) unpack.on('exit', function (code) { - update_ref(sha) todo-- if (todo <= 0) { // These writes are actually necessary for git to finish @@ -0,0 +1,129 @@ +#!/usr/bin/env node + +var spawn = require('child_process').spawn + +// Returns a process running `git ls-remote <url>` that calls `with_ref` on +// each parsed reference. The url may point to a local repository. +function ls (url, with_ref) { + var ls = spawn('git', ['ls-remote', url]) + ls.stdout.on('data', function (lines) { + lines.toString().split('\n').forEach(function (line) { + if (!line || line === '') { + return + } + line = line.split('\t') + var sha = line[0] + var branch = line[1] + if (sha.length !== 40) { + console.warn('[git ls-remote] expected a 40-byte sha: ' + sha + '\n') + console.warn('[git ls-remote] on line: ' + line.join('\t')) + } + with_ref(sha, branch) + }) + }) + return ls +} + +function pad4 (num) { + num = num.toString(16) + while (num.length < 4) { + num = '0' + num + } + return num +} + +// Invokes `$ git-upload-pack --strict <dir>`, communicates haves and wants and +// emits 'ready' when stdout becomes a pack file stream. +function upload_pack (dir, want, have) { + // reference: + // https://github.com/git/git/blob/b594c975c7e865be23477989d7f36157ad437dc7/Documentation/technical/pack-protocol.txt#L346-L393 + var upload = spawn('git-upload-pack', ['--strict', dir]) + writeln('want ' + want) + writeln() + if (have) { + writeln('have ' + have) + writeln() + } + writeln('done') + + // We want to read git's output one line at a time, and not read any more + // than we have to. That way, when we finish discussing wants and haves, we + // can pipe the rest of the output to a stream. + // + // We use `mode` to keep track of state and formulate responses. It returns + // `false` when we should stop reading. + var mode = list + upload.stdout.on('readable', function () { + while (true) { + var line = getline() + if (line === null) { + return // to wait for more output + } + if (!mode(line)) { + upload.stdout.removeAllListeners('readable') + upload.emit('ready') + return + } + } + }) + + var getline_len = null + // Extracts exactly one line from the stream. Uses `getline_len` in case the + // whole line could not be read. + function getline () { + // Format: '####line' where '####' represents the length of 'line' in hex. + if (!getline_len) { + getline_len = upload.stdout.read(4) + if (getline_len === null) { + return null + } + getline_len = parseInt(getline_len, 16) + } + + if (getline_len === 0) { + return '' + } + + // Subtract by the four we just read, and the terminating newline. + var line = upload.stdout.read(getline_len - 4 - 1) + if (!line) { + return null + } + getline_len = null + upload.stdout.read(1) // And discard the newline. + return line.toString() + } + + // First, the server lists the refs it has, but we already know from + // `git ls-remote`, so wait for it to signal the end. + function list (line) { + if (line === '') { + mode = have ? ack_objects_continue : wait_for_nak + } + return true + } + + // If we only gave wants, git should respond with 'NAK', then the pack file. + function wait_for_nak (line) { + return line !== 'NAK' + } + + // With haves, we wait for 'ACK', but only if not ending in 'continue'. + function ack_objects_continue (line) { + return !(line.search(/^ACK/) !== -1 && line.search(/continue$/) === -1) + } + + // Writes one line to stdin so git-upload-pack can understand. + function writeln (line) { + if (line) { + var len = pad4(line.length + 4 + 1) // Add one for the newline. + upload.stdin.write(len + line + '\n') + } else { + upload.stdin.write('0000') + } + } + + return upload +} + +module.exports = {ls: ls, upload_pack: upload_pack} diff --git a/gittorrentd b/gittorrentd index 0ac810d..d498590 100755 --- a/gittorrentd +++ b/gittorrentd @@ -15,6 +15,7 @@ var ut_metadata = require('ut_metadata') var WebTorrent = require('webtorrent') var zeroFill = require('zero-fill') var config = require('./config') +var git = require('./git') // BitTorrent client version string (used in peer ID). // Generated from package.json major and minor version. For example: @@ -71,6 +72,8 @@ function bpad (n, buf) { } } +var head = '' + dht.on('ready', function () { // Spider all */.git dirs and announce all refs. var repos = glob.sync('*/{,.git/}git-daemon-export-ok', {strict: false}) @@ -83,55 +86,34 @@ dht.on('ready', function () { var reponame = repo.replace(/\/.git\/$/, '') userProfile.repositories[reponame] = {} - var upload = spawn('git-upload-pack', ['--strict', repo]) - upload.stdout.on('data', function (line) { - var lines = line.toString().split('\n') - lines.forEach(function (line) { - var arr = line.toString().split(' ') - if (arr.length < 2) { - return - } - var sha = arr[0].toString() - // First four chars are git-upload-pack's length-of-line metadata. - sha = sha.substring(4) - if (arr.length == 2) { - var ref = arr[1].toString() - var branch = ref.match(/^refs\/heads\/(.*)/) - // FIXME: Can't pull in too many branches. - // if (!branch) { - // branch = ref.match(/^refs\/remotes\/(.*)/) - // } - if (branch) { - userProfile.repositories[reponame][ref] = sha - } - if (branch && !announcedRefs[sha]) { - branch = branch[1] - console.log('Announcing ' + sha + ' for ' + branch + ' on repo ' + repo) - announcedRefs[sha] = repo - dht.announce(sha, config.dht.announce, function (err) { - if (err !== null) { - console.log('Announced ' + sha) - } - }) + var ls = git.ls(repo, function (sha, ref) { + // FIXME: Can't pull in too many branches, so only do heads for now. + if (ref !== 'HEAD' && !ref.match(/^refs\/heads\//)) { + return + } + if (ref === 'refs/heads/master') { + head = sha + } + userProfile.repositories[reponame][ref] = sha + if (!announcedRefs[sha]) { + console.log('Announcing ' + sha + ' for ' + ref + ' on repo ' + repo) + announcedRefs[sha] = repo + dht.announce(sha, config.dht.announce, function (err) { + if (err !== null) { + console.log('Announced ' + sha) } - } else if (arr.length > 2 && arr[1].search(/^HEAD/) !== -1) { - // Probably the first line; line[0] has the hash, line[1] has HEAD, - // and beyond are the supported features. - userProfile.repositories[reponame]['HEAD'] = sha - } - }) - // Callback counting for repos + }) + } + }) + ls.stdout.on('end', function () { count-- if (count <= 0) { publish_mutable_key() } }) - upload.stdout.on('end', function () { - console.log('end') - }) - upload.on('exit', function (code) { - if (code !== 0) { - die('Failed: ' + code) + ls.on('exit', function (err) { + if (err) { + die(err) } }) }) @@ -171,19 +153,23 @@ dht.on('ready', function () { wire.handshake(new Buffer(infoHash), new Buffer(myPeerId)) }) wire.ut_gittorrent.on('generatePack', function (sha) { - console.error('calling git pack-objects') - var filename = sha + '.pack' - var stream = fs.createWriteStream(filename) + console.error('calling git pack-objects for ' + sha) if (!announcedRefs[sha]) { console.error('Asked for an unknown sha: ' + sha) return } var directory = announcedRefs[sha] - var pack = spawn('git', ['pack-objects', '--revs', '--thin', '--stdout', '--delta-base-offset'], {cwd: directory}) - pack.on('close', function (code) { - if (code !== 0) { - console.error('git pack-objects process exited with code ' + code) - } else { + var have = null + if (sha !== head) { + have = head + } + var pack = git.upload_pack(directory, sha, have) + pack.stderr.pipe(process.stderr) + pack.on('ready', function () { + var filename = sha + '.pack' + var stream = fs.createWriteStream(filename) + pack.stdout.pipe(stream) + stream.on('close', function () { console.error('Finished writing ' + filename) var webtorrent = new WebTorrent({ dht: {bootstrap: config.dht.bootstrap}, @@ -193,17 +179,13 @@ dht.on('ready', function () { console.error(torrent.infoHash) wire.ut_gittorrent.sendTorrent(torrent.infoHash) }) - } - }) - pack.stdout.pipe(stream) - pack.stderr.on('data', function (data) { - console.error(data.toString()) + }) }) - pack.on('exit', function () { - console.log('exited') + pack.on('exit', function (code) { + if (code !== 0) { + console.error('git-upload-pack process exited with code ' + code) + } }) - pack.stdin.write(sha + '\n') - pack.stdin.write('--not\n\n') }) }).listen(config.dht.announce) }) |