monotone

monotone Mtn Source Tree

Root/git_export.cc

1// -*- mode: C++; c-file-style: "gnu"; indent-tabs-mode: nil -*-
2// vim:sw=2:
3// Copyright (C) 2005 Petr Baudis <pasky@suse.cz>
4// all rights reserved.
5// licensed to the public under the terms of the GNU GPL (>= 2)
6// see the file COPYING for details
7
8// Sponsored by Google's Summer of Code and SuSE
9
10// This whole thing needs massive cleanup and codesharing with git_import.cc.
11
12#include <algorithm>
13#include <iostream>
14#include <fstream>
15#include <iterator>
16#include <list>
17#include <map>
18#include <set>
19#include <sstream>
20#include <stack>
21#include <stdexcept>
22#include <string>
23#include <vector>
24#include <queue>
25#include <stdlib.h>
26
27#ifndef WIN32
28
29#include <unistd.h>
30#include <sys/stat.h> // mkdir()
31
32#include <stdio.h>
33#include <string.h> // strdup(), woo-hoo!
34
35#include <boost/shared_ptr.hpp>
36#include <boost/scoped_ptr.hpp>
37#include <boost/lexical_cast.hpp>
38#include <boost/tokenizer.hpp>
39
40#include <boost/filesystem/path.hpp>
41#include <boost/filesystem/operations.hpp>
42#include <boost/filesystem/convenience.hpp>
43
44#include "botan/botan.h"
45
46#include "app_state.hh"
47#include "cert.hh"
48#include "constants.hh"
49#include "database.hh"
50#include "file_io.hh"
51#include "git.hh"
52#include "git_export.hh"
53#include "keys.hh"
54#include "manifest.hh"
55#include "mkstemp.hh"
56#include "packet.hh"
57#include "sanity.hh"
58#include "transforms.hh"
59#include "ui.hh"
60
61using namespace std;
62using boost::shared_ptr;
63using boost::scoped_ptr;
64
65struct
66git_tree_entry
67{
68 bool execute;
69 git_object_id blob_id;
70 file_path path;
71};
72
73struct git_history;
74
75// staging area for exporting
76struct
77git_staging
78{
79 system_path path;
80 git_history *git;
81
82 git_staging(git_history *g);
83 ~git_staging();
84
85 git_object_id blob_save(data const &blob);
86 git_object_id tree_save(set<shared_ptr<git_tree_entry> > const &entries);
87 git_object_id commit_save(git_object_id const &tree,
88 set<git_object_id> const &parents,
89 git_person const &author,
90 boost::posix_time::ptime const &atime,
91 git_person const &committer,
92 boost::posix_time::ptime const &ctime,
93 data const &logmsg);
94
95private:
96 system_path index_file;
97};
98
99
100// This is pretty much the _reverse_ of git_import.cc's git_history!
101struct
102git_history
103{
104 git_staging staging;
105
106 map<revision_id, git_object_id> commitmap;
107 map<manifest_id, git_object_id> treemap;
108 map<file_id, git_object_id> filemap;
109
110 ticker n_revs;
111 ticker n_objs;
112
113 string branch;
114
115 git_history();
116};
117
118
119/*** The raw GIT interface */
120
121// XXX: Code duplication with git_db::load_revs().
122static void
123get_gitrev_ancestry(git_object_id revision, set<git_object_id> &ancestry)
124{
125 filebuf fb;
126 capture_git_cmd_output(F("git-rev-list %s") % revision(), fb);
127 istream stream(&fb);
128
129 stack<git_object_id> st;
130 while (!stream.eof())
131 {
132 char revbuf[41];
133 stream.getline(revbuf, 41);
134 if (strlen(revbuf) < 40)
135 continue;
136 L(F("noted revision %s") % revbuf);
137 ancestry.insert(git_object_id(string(revbuf)));
138 }
139 L(F("Loaded all revisions"));
140}
141
142
143git_staging::git_staging(git_history *g)
144 : git(g)
145{
146 // Make a temporary staging directory:
147 char *tmpdir = getenv("TMPDIR");
148 if (!tmpdir)
149 tmpdir = "/tmp";
150 string tmpfile(tmpdir);
151 tmpfile += "/mtexport.XXXXXX";
152 int fd = monotone_mkstemp(tmpfile);
153
154 // Hope for the racy best.
155 close(fd);
156 delete_file(system_path(tmpfile));
157
158 E(mkdir(tmpfile.c_str(), 0700) == 0, F("mkdir(%s) failed") % tmpfile);
159
160
161 path = system_path(tmpfile);
162 index_file = path / "index";
163}
164
165git_staging::~git_staging()
166{
167 rmdir(path.as_external().c_str());
168}
169
170git_object_id
171git_staging::blob_save(data const &blob)
172{
173 ++git->n_objs;
174
175 string strpath = path.as_external();
176 string blobpath = (path / "blob").as_external();
177 {
178 ofstream file(blobpath.c_str(), ios_base::out | ios_base::trunc | ios_base::binary);
179 E(file, F("cannot open file %s for writing") % blobpath);
180 Botan::Pipe pipe(new Botan::DataSink_Stream(file));
181 pipe.process_msg(blob());
182 }
183
184 set_git_env("GIT_INDEX_FILE", index_file.as_external());
185
186 string cmdline("cd '" + strpath + "' && git-update-cache --add blob");
187 L(F("Invoking: %s") % cmdline);
188 E(system(cmdline.c_str()) == 0, F("Adding '%s' failed") % blobpath);
189
190 filebuf fb;
191 capture_git_cmd_output(F("cd '%s' && git-ls-files --stage") % strpath, fb);
192 istream stream(&fb);
193 string line;
194 stream_grabline(stream, line);
195 E(line.length() >= 40, F("Invalid generated index, containing: '%s'") % line);
196 git_object_id gitoid(line.substr(line.find(" ") + 1, 40));
197
198 delete_file(index_file);
199 delete_file(path / "blob");
200
201 return gitoid;
202}
203
204git_object_id
205git_staging::tree_save(set<shared_ptr<git_tree_entry> > const &entries)
206{
207 ++git->n_objs;
208
209 set_git_env("GIT_INDEX_FILE", index_file.as_external());
210
211 string cmdline("git-update-cache --add ");
212 for (set<shared_ptr<git_tree_entry> >::const_iterator i = entries.begin();
213 i != entries.end(); ++i)
214 {
215 cmdline += "--cacheinfo ";
216 cmdline += (*i)->execute ? "777" : "666";
217 cmdline += " " + (*i)->blob_id();
218 // FIXME: Quote!
219 cmdline += " '" + (*i)->path.as_external() + "' ";
220 }
221 L(F("Invoking: %s") % cmdline);
222 E(system(cmdline.c_str()) == 0, F("Writing tree index failed"));
223
224 filebuf fb;
225 capture_git_cmd_output(F("git-write-tree"), fb);
226 istream stream(&fb);
227 string line;
228 stream_grabline(stream, line);
229 E(line.length() == 40, F("Invalid git-write-tree output: %s") % line);
230 git_object_id gittid(line);
231
232 delete_file(index_file);
233
234 return gittid;
235}
236
237git_object_id
238git_staging::commit_save(git_object_id const &tree,
239 set<git_object_id> const &parents,
240 git_person const &author,
241 boost::posix_time::ptime const &atime,
242 git_person const &committer,
243 boost::posix_time::ptime const &ctime,
244 data const &logmsg)
245{
246 ++git->n_revs;
247 ++git->n_objs;
248
249 L(F("Author: %s/%s, Committer: %s/%s") % author.name % author.email % committer.name % committer.email);
250 set_git_env("GIT_AUTHOR_NAME", author.name);
251 set_git_env("GIT_AUTHOR_EMAIL", author.email);
252 set_git_env("GIT_AUTHOR_DATE", to_iso_extended_string(atime));
253 set_git_env("GIT_COMMITTER_NAME", committer.name);
254 set_git_env("GIT_COMMITTER_EMAIL", committer.email);
255 set_git_env("GIT_COMMITTER_DATE", to_iso_extended_string(ctime));
256 L(F("Logmsg: %s") % logmsg());
257 data mylogmsg = data(logmsg() + "\n");
258
259 string cmdline("git-commit-tree " + tree() + " ");
260 for (set<git_object_id>::const_iterator i = parents.begin();
261 i != parents.end(); ++i)
262 {
263 cmdline += "-p " + (*i)() + " ";
264 }
265 filebuf fb;
266 capture_git_cmd_io(F("%s") % cmdline, mylogmsg, fb);
267 istream stream(&fb);
268 string line;
269 stream_grabline(stream, line);
270 E(line.length() == 40, F("Invalid git-commit-tree output: %s") % line);
271 git_object_id gitcid(line);
272 return gitcid;
273}
274
275
276static git_object_id
277export_git_blob(git_history &git, app_state &app, file_id fid)
278{
279 L(F("Exporting file '%s'") % fid.inner());
280
281 map<file_id, git_object_id>::const_iterator i = git.filemap.find(fid);
282 if (i != git.filemap.end())
283 {
284 return i->second;
285 }
286
287 file_data fdata;
288 app.db.get_file_version(fid, fdata);
289 git_object_id gitbid = git.staging.blob_save(fdata.inner());
290 git.filemap.insert(make_pair(fid, gitbid));
291 return gitbid;
292}
293
294static git_object_id
295export_git_tree(git_history &git, app_state &app, manifest_id mid)
296{
297 L(F("Exporting tree '%s'") % mid.inner());
298
299 map<manifest_id, git_object_id>::const_iterator i = git.treemap.find(mid);
300 if (i != git.treemap.end())
301 {
302 return i->second;
303 }
304
305 manifest_map manifest;
306 app.db.get_manifest(mid, manifest);
307
308 attr_map attrs;
309 read_attr_map_from_db(manifest, attrs, app);
310
311 set<shared_ptr<git_tree_entry> > tree;
312 for (manifest_map::const_iterator i = manifest.begin();
313 i != manifest.end(); ++i)
314 {
315 shared_ptr<git_tree_entry> entry(new git_tree_entry);
316 entry->blob_id = export_git_blob(git, app, manifest_entry_id(*i));
317 entry->path = manifest_entry_path(*i);
318
319 string attrval;
320 entry->execute = false;
321 if (find_in_attr_map(attrs, entry->path, "execute", attrval))
322entry->execute = (attrval == "true");
323
324 L(F("Queuing '%s' [%s:%s] (%d)") % entry->path % manifest_entry_id(*i)
325% entry->blob_id() % entry->execute);
326 tree.insert(entry);
327 }
328
329 git_object_id gittid = git.staging.tree_save(tree);
330 git.treemap.insert(make_pair(mid, gittid));
331 return gittid;
332}
333
334
335static bool
336has_cert(app_state &app, revision_id rid, cert_name name, string content)
337{
338 L(F("Has cert '%s' of value '%s'?") % name % content);
339
340 vector< revision<cert> > certs;
341 app.db.get_revision_certs(rid, name, certs);
342 erase_bogus_certs(certs, app);
343 for (vector< revision<cert> >::const_iterator i = certs.begin();
344 i != certs.end(); ++i)
345 {
346 cert_value tv;
347 decode_base64(i->inner().value, tv);
348 if (content == tv())
349return true;
350 }
351 L(F("... nope"));
352 return false;
353}
354
355static void
356load_cert(app_state &app, revision_id rid, cert_name name, string &content)
357{
358 L(F("Loading cert '%s'") % name);
359
360 vector< revision<cert> > certs;
361 app.db.get_revision_certs(rid, name, certs);
362 erase_bogus_certs(certs, app);
363 if (certs.begin() == certs.end())
364 return;
365
366 cert_value tv;
367 decode_base64(certs.begin()->inner().value, tv);
368 content = tv();
369
370 L(F("... '%s'") % content);
371}
372
373static void
374historical_monorev_to_gitrev(git_history &git, app_state &app,
375 revision_id rid, git_object_id &gitrid)
376{
377 cert_name commitid_name(gitcommit_id_cert_name);
378 string commitid;
379 load_cert(app, rid, commitid_name, commitid);
380 E(!commitid.empty(), F("Current commit's parent %s was not imported yet?!") % rid.inner());
381 gitrid = git_object_id(commitid);
382}
383
384static bool
385export_git_revision(git_history &git, app_state &app, revision_id rid, git_object_id &gitcid)
386{
387 L(F("Exporting commit '%s'") % rid.inner());
388
389 cert_name branch_name(branch_cert_name);
390
391#if 0
392 // See 88020b6892b125ad3ac5888cc90c2df4d33ab476 in Monotone's
393 // revision history.
394 if (!has_cert(app, rid, branch_name, git.branch))
395 {
396 L(F("Skipping, not on my branch."));
397 return false;
398 }
399#endif
400
401 revision_set rev;
402 app.db.get_revision(rid, rev);
403 git_object_id gittid = export_git_tree(git, app, rev.new_manifest);
404
405 set<git_object_id> parents;
406 for (edge_map::const_iterator e = rev.edges.begin();
407 e != rev.edges.end(); ++e)
408 {
409 if (null_id(edge_old_revision(e)))
410continue;
411
412 L(F("Considering edge %s -> %s") % rid.inner() % edge_old_revision(e).inner());
413 map<revision_id, git_object_id>::const_iterator i;
414 i = git.commitmap.find(edge_old_revision(e));
415 git_object_id parent_gitcid;
416 if (i != git.commitmap.end())
417 {
418 parent_gitcid = i->second;
419} else {
420 historical_monorev_to_gitrev(git, app, edge_old_revision(e), parent_gitcid);
421}
422 parents.insert(parent_gitcid);
423 }
424
425 cert_name author_name(author_cert_name);
426 cert_name date_name(date_cert_name);
427 cert_name committer_name(gitcommit_committer_cert_name);
428 cert_name changelog_name(changelog_cert_name);
429
430 git_person author;
431 load_cert(app, rid, author_name, author.email);
432
433 boost::posix_time::ptime atime;
434 string atimestr;
435 load_cert(app, rid, date_name, atimestr);
436 atime = string_to_datetime(atimestr);
437
438 git_person committer;
439 boost::posix_time::ptime ctime;
440 string commitline;
441 load_cert(app, rid, committer_name, commitline);
442 if (!commitline.empty())
443 {
444 committer.name = commitline.substr(0, commitline.find("<") - 1);
445 commitline.erase(0, commitline.find("<") + 1);
446 committer.email = commitline.substr(0, commitline.find(">"));
447 commitline.erase(0, commitline.find(">") + 2);
448 ctime = boost::posix_time::from_iso_string(commitline.substr(0, commitline.find(" ")));
449 }
450 else
451 {
452 ctime = boost::posix_time::second_clock::universal_time();
453 }
454
455 string logmsg;
456 load_cert(app, rid, changelog_name, logmsg);
457
458 gitcid = git.staging.commit_save(gittid, parents,
459 author, atime,
460 committer, ctime,
461 data(logmsg));
462 git.commitmap.insert(make_pair(rid, gitcid));
463
464 packet_db_writer dbw(app);
465 put_simple_revision_cert(rid, gitcommit_id_cert_name, gitcid(), app, dbw);
466
467 return true;
468}
469
470
471// Like database::get_revision_ancestry() but for just a single branch
472static void
473get_branch_ancestry(string const &branch, app_state &app,
474 set<revision_id> &list)
475{
476 queue<revision_id> frontier;
477
478 set<revision_id> heads;
479 get_branch_heads(branch, app, heads);
480 for (set<revision_id>::const_iterator i = heads.begin();
481 i != heads.end(); ++i)
482 frontier.push(*i);
483
484 while (!frontier.empty())
485 {
486 revision_id rid = frontier.front(); frontier.pop();
487
488 if (list.find(rid) != list.end())
489 continue;
490 list.insert(rid);
491
492 revision_set rev;
493 app.db.get_revision(rid, rev);
494
495 for (edge_map::const_iterator e = rev.edges.begin();
496 e != rev.edges.end(); ++e)
497 {
498 if (!null_id(edge_old_revision(e)))
499 frontier.push(edge_old_revision(e));
500 }
501 }
502}
503
504
505static void
506add_gitrevs_descendants(git_history &git, app_state &app,
507 set<revision_id> &list, set<git_object_id> gitrevs)
508{
509 queue<revision_id> frontier;
510 set<revision_id> seen;
511
512 set<revision_id> heads;
513 get_branch_heads(git.branch, app, heads);
514 for (set<revision_id>::const_iterator i = heads.begin();
515 i != heads.end(); ++i)
516 frontier.push(*i);
517
518 while (!frontier.empty())
519 {
520 revision_id rid = frontier.front(); frontier.pop();
521
522 if (seen.find(rid) != seen.end())
523 continue;
524 seen.insert(rid);
525
526 L(F("descendant search: Considering %s") % rid.inner());
527 revision_set rev;
528 app.db.get_revision(rid, rev);
529
530 vector<revision<cert> > certs;
531 app.db.get_revision_certs(rid, gitcommit_id_cert_name, certs);
532 for (vector<revision<cert> >::const_iterator c = certs.begin();
533 c != certs.end(); c++)
534 {
535 // This is a GIT commit, then.
536 cert_value cv;
537 decode_base64(c->inner().value, cv);
538 git_object_id gitoid = cv();
539 L(F("... git ID %s") % gitoid());
540
541 if (gitrevs.find(cv()) != gitrevs.end())
542 continue;
543 }
544 list.insert(rid);
545
546 for (edge_map::const_iterator e = rev.edges.begin();
547 e != rev.edges.end(); ++e)
548 {
549 if (!null_id(edge_old_revision(e)))
550 frontier.push(edge_old_revision(e));
551 }
552 }
553}
554
555
556git_history::git_history()
557 : staging(this), n_revs("revisions", "r", 1), n_objs("objects", "o", 4)
558{
559}
560
561
562void
563export_git_repo(system_path const & gitrepo,
564 string const &headname_,
565 app_state & app)
566{
567 {
568 // early short-circuit to avoid failure after lots of work
569 // We need to write to the repository to save the gitcommit-id certs
570 // to faciliate incremental exports and export-imports.
571 rsa_keypair_id key;
572 N(guess_default_key(key,app),
573 F("no unique private key for cert construction"));
574 require_password(key, app);
575 }
576
577 require_path_is_directory(gitrepo,
578 F("repo %s does not exist") % gitrepo,
579 F("repo %s is not a directory") % gitrepo);
580
581 N(app.branch_name() != "", F("need base --branch argument for exporting"));
582
583 set<revision_id> heads;
584 get_branch_heads(app.branch_name(), app, heads);
585 N(heads.size() == 1, F("the to-be-exported branch has to have exactly one head"));
586
587 set_git_env("GIT_DIR", gitrepo.as_external());
588 git_history git;
589 git.branch = app.branch_name();
590
591 string headname(headname_.empty() ? "master" : headname_);
592 system_path headpath(gitrepo / "refs/heads" / headname);
593
594 // Nothing shall disturb us!
595 transaction_guard guard(app.db);
596 app.db.ensure_open();
597
598 set<revision_id> filter;
599 toposort_filter filtertype = topo_all;
600 if (file_exists(headpath))
601 {
602 ifstream file(headpath.as_external().c_str(), ios_base::in);
603 N(file, F("cannot open file %s for reading") % headpath);
604 string line;
605 stream_grabline(file, line);
606 I(line.length() == 40);
607 git_object_id gitrev(line);
608 set<git_object_id> ancestry;
609 get_gitrev_ancestry(gitrev, ancestry);
610 add_gitrevs_descendants(git, app, filter, ancestry);
611 filtertype = topo_include;
612
613 try
614 {
615 revision_id rev;
616 historical_gitrev_to_monorev(git.branch, NULL, app, gitrev, rev);
617 }
618 catch (std::exception &e)
619 {
620 N(false, F("head %s is not subset of our tree; perhaps import first?") % headname);
621}
622 }
623 else
624 {
625 get_branch_ancestry(git.branch, app, filter);
626 filtertype = topo_include;
627 }
628
629 vector<revision_id> revlist; revlist.clear();
630 // fill revlist with all the revisions, toposorted
631 toposort(filter, revlist, app, filtertype);
632 //reverse(revlist.begin(), revlist.end());
633
634 git_object_id gitcid;
635 for (vector<revision_id>::const_iterator i = revlist.begin();
636 i != revlist.end(); ++i)
637 {
638 if (null_id(*i))
639continue;
640
641 ui.set_tick_trailer((*i).inner()());
642 export_git_revision(git, app, *i, gitcid);
643 }
644 ui.set_tick_trailer("");
645
646 guard.commit();
647
648 ofstream file(headpath.as_external().c_str(),
649 ios_base::out | ios_base::trunc);
650 N(file, F("cannot open file %s for writing") % headpath);
651 file << gitcid() << endl;
652
653 return;
654}
655
656#else // WIN32
657
658void
659export_git_repo(system_path const & gitrepo,
660 app_state & app)
661{
662 E("git export not supported on win32");
663}
664
665#endif

Archive Download this file

Branches

Tags

Quick Links:     www.monotone.ca    -     Downloads    -     Documentation    -     Wiki    -     Code Forge    -     Build Status