monotone

monotone Mtn Source Tree

Root/src/database.cc

1// Copyright (C) 2010 Stephen Leake <stephen_leake@stephe-leake.org>
2// Copyright (C) 2002 Graydon Hoare <graydon@pobox.com>
3//
4// This program is made available under the GNU GPL version 2.0 or
5// greater. See the accompanying file COPYING for details.
6//
7// This program is distributed WITHOUT ANY WARRANTY; without even the
8// implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
9// PURPOSE.
10
11#include "base.hh"
12#include <algorithm>
13#include <deque>
14#include <fstream>
15#include <iterator>
16#include <list>
17#include <numeric>
18#include <set>
19#include <sstream>
20#include "vector.hh"
21
22#include <string.h>
23#include <boost/bind.hpp>
24#include <boost/shared_ptr.hpp>
25#include <boost/tuple/tuple.hpp>
26#include <boost/tuple/tuple_comparison.hpp>
27
28#include <botan/botan.h>
29#include <botan/rsa.h>
30#include <botan/pem.h>
31#include <botan/look_pk.h>
32#include "lazy_rng.hh"
33
34#include <sqlite3.h>
35
36#include "lexical_cast.hh"
37
38#include "app_state.hh"
39#include "cert.hh"
40#include "project.hh"
41#include "cleanup.hh"
42#include "constants.hh"
43#include "dates.hh"
44#include "database.hh"
45#include "hash_map.hh"
46#include "keys.hh"
47#include "platform-wrapped.hh"
48#include "revision.hh"
49#include "safe_map.hh"
50#include "sanity.hh"
51#include "migration.hh"
52#include "simplestring_xform.hh"
53#include "transforms.hh"
54#include "ui.hh" // tickers
55#include "vocab.hh"
56#include "vocab_cast.hh"
57#include "xdelta.hh"
58#include "epoch.hh"
59#include "graph.hh"
60#include "roster.hh"
61#include "roster_delta.hh"
62#include "rev_height.hh"
63#include "vocab_hash.hh"
64#include "globish.hh"
65#include "work.hh"
66#include "lua_hooks.hh"
67#include "outdated_indicator.hh"
68#include "lru_writeback_cache.hh"
69#include "char_classifiers.hh"
70
71// defined in schema.c, generated from schema.sql:
72extern char const schema_constant[];
73
74// this file defines a public, typed interface to the database.
75// the database class encapsulates all knowledge about sqlite,
76// the schema, and all SQL statements used to access the schema.
77//
78// see file schema.sql for the text of the schema.
79
80using std::deque;
81using std::istream;
82using std::make_pair;
83using std::map;
84using std::multimap;
85using std::ostream;
86using std::pair;
87using std::remove_if;
88using std::set;
89using std::sort;
90using std::string;
91using std::vector;
92using std::accumulate;
93
94using boost::shared_ptr;
95using boost::shared_dynamic_cast;
96using boost::lexical_cast;
97using boost::get;
98using boost::tuple;
99using boost::lexical_cast;
100
101using Botan::PK_Encryptor;
102using Botan::PK_Verifier;
103using Botan::SecureVector;
104using Botan::X509_PublicKey;
105using Botan::RSA_PublicKey;
106using Botan::get_pk_encryptor;
107
108int const one_row = 1;
109int const one_col = 1;
110int const any_rows = -1;
111int const any_cols = -1;
112
113namespace
114{
115 struct query_param
116 {
117 enum arg_type { text, blob, int64 };
118 arg_type type;
119 string string_data;
120 u64 int_data;
121 };
122
123 query_param
124 text(string const & txt)
125 {
126 MM(txt);
127 for (string::const_iterator i = txt.begin();
128 i != txt.end(); ++i)
129 {
130 I(*i >= 10 && *i < 127);
131 }
132 query_param q = {
133 query_param::text,
134 txt,
135 0,
136 };
137 return q;
138 }
139
140 query_param
141 blob(string const & blb)
142 {
143 query_param q = {
144 query_param::blob,
145 blb,
146 0,
147 };
148 return q;
149 }
150
151 query_param
152 int64(u64 const & num)
153 {
154 query_param q = {
155 query_param::int64,
156 "",
157 num,
158 };
159 return q;
160 }
161
162 struct query
163 {
164 explicit query(string const & cmd)
165 : sql_cmd(cmd)
166 {}
167
168 query()
169 {}
170
171 query & operator %(query_param const & qp)
172 {
173 args.push_back(qp);
174 return *this;
175 }
176
177 vector<query_param> args;
178 string sql_cmd;
179 };
180
181 typedef vector< vector<string> > results;
182
183 struct statement
184 {
185 statement() : count(0), stmt(0, sqlite3_finalize) {}
186 int count;
187 cleanup_ptr<sqlite3_stmt*, int> stmt;
188 };
189
190 struct roster_size_estimator
191 {
192 unsigned long operator() (cached_roster const & cr)
193 {
194 I(cr.first);
195 I(cr.second);
196 // do estimate using a totally made up multiplier, probably wildly off
197 return (cr.first->all_nodes().size()
198 * constants::db_estimated_roster_node_sz);
199 }
200 };
201
202 struct datasz
203 {
204 unsigned long operator()(data const & t) { return t().size(); }
205 };
206
207 enum open_mode { normal_mode = 0,
208 schema_bypass_mode,
209 format_bypass_mode,
210 cache_bypass_mode };
211
212 typedef hashmap::hash_map<revision_id, set<revision_id> > parent_id_map;
213 typedef hashmap::hash_map<revision_id, rev_height> height_map;
214
215 typedef hashmap::hash_map<key_id,
216 pair<shared_ptr<Botan::PK_Verifier>,
217 shared_ptr<Botan::RSA_PublicKey> >
218 > verifier_cache;
219
220} // anonymous namespace
221
222class database_impl
223{
224 friend class database;
225
226 // for scoped_ptr's sake
227public:
228 explicit database_impl(system_path const & f, db_type t,
229 system_path const & roster_cache_performance_log);
230 ~database_impl();
231
232private:
233
234 //
235 // --== Opening the database and schema checking ==--
236 //
237 system_path filename;
238 db_type type;
239 struct sqlite3 * __sql;
240
241 void install_functions();
242 struct sqlite3 * sql(enum open_mode mode = normal_mode);
243
244 void check_filename();
245 void check_db_exists();
246 void check_db_nonexistent();
247 void open();
248 void close();
249 void check_format();
250 void check_caches();
251
252 bool table_has_data(string const & name);
253
254 //
255 // --== Basic SQL interface and statement caching ==--
256 //
257 map<string, statement> statement_cache;
258
259 void fetch(results & res,
260 int const want_cols, int const want_rows,
261 query const & q);
262 void execute(query const & q);
263
264 bool table_has_entry(id const & key, string const & column,
265 string const & table);
266
267 //
268 // --== Generic database metadata gathering ==--
269 //
270 string count(string const & table);
271 string space(string const & table,
272 string const & concatenated_columns,
273 u64 & total);
274 unsigned int page_size();
275 unsigned int cache_size();
276
277 //
278 // --== Transactions ==--
279 //
280 int transaction_level;
281 bool transaction_exclusive;
282 void begin_transaction(bool exclusive);
283 void commit_transaction();
284 void rollback_transaction();
285 friend class conditional_transaction_guard;
286
287 struct roster_writeback_manager
288 {
289 database_impl & imp;
290 roster_writeback_manager(database_impl & imp) : imp(imp) {}
291 void writeout(revision_id const &, cached_roster const &);
292 };
293 LRUWritebackCache<revision_id, cached_roster,
294 roster_size_estimator, roster_writeback_manager>
295 roster_cache;
296
297 bool have_delayed_file(file_id const & id);
298 void load_delayed_file(file_id const & id, file_data & dat);
299 void cancel_delayed_file(file_id const & id);
300 void drop_or_cancel_file(file_id const & id);
301 void schedule_delayed_file(file_id const & id, file_data const & dat);
302
303 map<file_id, file_data> delayed_files;
304 size_t delayed_writes_size;
305
306 void flush_delayed_writes();
307 void clear_delayed_writes();
308 void write_delayed_file(file_id const & new_id,
309 file_data const & dat);
310
311 void write_delayed_roster(revision_id const & new_id,
312 roster_t const & roster,
313 marking_map const & marking);
314
315 //
316 // --== Reading/writing delta-compressed objects ==--
317 //
318
319 // "do we have any entry for 'ident' that is a base version"
320 bool roster_base_stored(revision_id const & ident);
321 bool roster_base_available(revision_id const & ident);
322
323 // "do we have any entry for 'ident' that is a delta"
324 bool delta_exists(file_id const & ident,
325 file_id const & base,
326 string const & table);
327
328 bool file_or_manifest_base_exists(id const & ident,
329 std::string const & table);
330
331 void get_file_or_manifest_base_unchecked(id const & new_id,
332 data & dat,
333 string const & table);
334 void get_file_or_manifest_delta_unchecked(id const & ident,
335 id const & base,
336 delta & del,
337 string const & table);
338 void get_roster_base(revision_id const & ident,
339 roster_t & roster, marking_map & marking);
340 void get_roster_delta(id const & ident,
341 id const & base,
342 roster_delta & del);
343
344 friend struct file_and_manifest_reconstruction_graph;
345 friend struct roster_reconstruction_graph;
346
347 LRUWritebackCache<id, data, datasz> vcache;
348
349 void get_version(id const & ident,
350 data & dat,
351 string const & data_table,
352 string const & delta_table);
353
354 void drop(id const & base,
355 string const & table);
356
357 void put_file_delta(file_id const & ident,
358 file_id const & base,
359 file_delta const & del);
360
361 void put_file_size(file_id const & ident,
362 file_data const & data);
363
364 void put_roster_delta(revision_id const & ident,
365 revision_id const & base,
366 roster_delta const & del);
367
368 //
369 // --== The ancestry graph ==--
370 //
371 void get_ids(string const & table, set<id> & ids);
372
373 //
374 // --== Rosters ==--
375 //
376 struct extractor;
377 struct file_content_extractor;
378 struct markings_extractor;
379 void extract_from_deltas(revision_id const & id, extractor & x);
380
381 height_map height_cache;
382 parent_id_map parent_cache;
383
384 //
385 // --== Keys ==--
386 //
387 void get_keys(string const & table, vector<key_name> & keys);
388
389 // cache of verifiers for public keys
390 verifier_cache verifiers;
391
392 //
393 // --== Certs ==--
394 //
395 // note: this section is ridiculous. please do something about it.
396 bool cert_exists(cert const & t,
397 string const & table);
398 void put_cert(cert const & t, string const & table);
399 void results_to_certs(results const & res,
400 vector<cert> & certs);
401 void results_to_certs(results const & res,
402 vector<pair<id, cert> > & certs);
403 void oldstyle_results_to_certs(results const & res,
404 vector<cert> & certs);
405
406 void get_certs(vector<cert> & certs,
407 string const & table);
408
409 void get_oldstyle_certs(id const & ident,
410 vector<cert> & certs,
411 string const & table);
412
413 void get_certs(id const & ident,
414 vector<cert> & certs,
415 string const & table);
416
417 void get_certs(cert_name const & name,
418 vector<cert> & certs,
419 string const & table);
420
421 void get_oldstyle_certs(cert_name const & name,
422 vector<cert> & certs,
423 string const & table);
424
425 void get_certs(id const & ident,
426 cert_name const & name,
427 vector<cert> & certs,
428 string const & table);
429
430 void get_certs(id const & ident,
431 cert_name const & name,
432 cert_value const & val,
433 vector<cert> & certs,
434 string const & table);
435
436 void get_certs(cert_name const & name,
437 cert_value const & val,
438 vector<pair<id, cert> > & certs,
439 string const & table);
440
441 outdated_indicator_factory cert_stamper;
442
443 void add_prefix_matching_constraint(string const & colname,
444 string const & prefix,
445 query & q);
446};
447
448#ifdef SUPPORT_SQLITE_BEFORE_3003014
449// SQLite versions up to and including 3.3.12 didn't have the hex() function
450void
451sqlite3_hex_fn(sqlite3_context *f, int nargs, sqlite3_value **args)
452{
453 if (nargs != 1)
454 {
455 sqlite3_result_error(f, "need exactly 1 arg to hex()", -1);
456 return;
457 }
458 string decoded;
459
460 // This operation may throw (un)recoverable_failure. We must intercept that
461 // and turn it into a call to sqlite3_result_error, or rollback will fail.
462 try
463 {
464 decoded = encode_hexenc(reinterpret_cast<char const *>(
465 sqlite3_value_text(args[0])), origin::database);
466 }
467 catch (recoverable_failure & e)
468 {
469 sqlite3_result_error(f, e.what(), -1);
470 return;
471 }
472 catch (unrecoverable_failure & e)
473 {
474 sqlite3_result_error(f, e.what(), -1);
475 return;
476 }
477
478 sqlite3_result_blob(f, decoded.data(), decoded.size(), SQLITE_TRANSIENT);
479}
480#endif
481
482database_impl::database_impl(system_path const & f, db_type t,
483 system_path const & roster_cache_performance_log) :
484 filename(f),
485 type(t),
486 __sql(NULL),
487 transaction_level(0),
488 roster_cache(constants::db_roster_cache_sz,
489 constants::db_roster_cache_min_count,
490 roster_writeback_manager(*this),
491 roster_cache_performance_log.as_external()),
492 delayed_writes_size(0),
493 vcache(constants::db_version_cache_sz, 1)
494{}
495
496database_impl::~database_impl()
497{
498 L(FL("statement cache statistics"));
499 L(FL("prepared %d statements") % statement_cache.size());
500
501 for (map<string, statement>::const_iterator i = statement_cache.begin();
502 i != statement_cache.end(); ++i)
503 L(FL("%d executions of %s") % i->second.count % i->first);
504 // trigger destructors to finalize cached statements
505 statement_cache.clear();
506
507 if (__sql)
508 close();
509}
510
511database_cache database::dbcache;
512
513database::database(app_state & app, database::dboptions d)
514 : opts(app.opts), lua(app.lua), dbopts(d)
515{
516 init();
517}
518
519database::database(options const & o, lua_hooks & l, database::dboptions d)
520 : opts(o), lua(l), dbopts(d)
521{
522 init();
523}
524
525void
526database::init()
527{
528 database_path_helper helper(lua);
529 system_path dbpath;
530 helper.get_database_path(opts, dbpath, dbopts);
531
532 // FIXME: for all :memory: databases an empty path is returned above, thus
533 // all requests for a :memory: database point to the same database
534 // implementation. This means we cannot use two different memory databases
535 // within the same monotone process
536 if (dbcache.find(dbpath) == dbcache.end())
537 {
538 L(FL("creating new database_impl instance for %s") % dbpath);
539 dbcache.insert(make_pair(dbpath, boost::shared_ptr<database_impl>(
540 new database_impl(dbpath, opts.dbname_type, opts.roster_cache_performance_log)
541 )));
542 }
543
544 imp = dbcache[dbpath];
545}
546
547database::~database()
548{}
549
550void
551database::reset_cache()
552{
553 dbcache.clear();
554}
555
556system_path
557database::get_filename()
558{
559 return imp->filename;
560}
561
562bool
563database::is_dbfile(any_path const & file)
564{
565 if (imp->type == memory_db)
566 return false;
567 system_path fn(file); // canonicalize
568 bool same = (imp->filename == fn);
569 if (same)
570 L(FL("'%s' is the database file") % file);
571 return same;
572}
573
574bool
575database::database_specified()
576{
577 return imp->type == memory_db || !imp->filename.empty();
578}
579
580void
581database::create_if_not_exists()
582{
583 imp->check_filename();
584 if (!file_exists(imp->filename))
585 {
586 P(F("initializing new database '%s'") % imp->filename);
587 initialize();
588 }
589}
590
591void
592database::check_is_not_rosterified()
593{
594 E(!imp->table_has_data("rosters"), origin::user,
595 F("this database already contains rosters"));
596}
597
598bool
599database_impl::table_has_data(string const & name)
600{
601 results res;
602 fetch(res, one_col, any_rows, query("SELECT 1 FROM " + name + " LIMIT 1"));
603 return !res.empty();
604}
605
606void
607database_impl::check_format()
608{
609 if (table_has_data("manifests"))
610 {
611 // The rosters and heights tables should be empty.
612 I(!table_has_data("rosters") && !table_has_data("heights"));
613
614 // they need to either changesetify or rosterify. which?
615 if (table_has_data("revisions"))
616 E(false, origin::no_fault,
617 F("database '%s' contains old-style revisions.\n"
618 "If you are a project leader or doing local testing:\n"
619 " see the file UPGRADE for instructions on upgrading.\n"
620 "If you are not a project leader:\n"
621 " wait for a leader to migrate project data, and then\n"
622 " pull into a fresh database.\n"
623 "Sorry about the inconvenience.")
624 % filename);
625 else
626 E(false, origin::no_fault,
627 F("database '%s' contains manifests but no revisions.\n"
628 "This is a very old database; it needs to be upgraded.\n"
629 "Please see 'http://wiki.monotone.ca/upgradefromchangesets/'\n"
630 "for details")
631 % filename);
632 }
633}
634
635void
636database_impl::check_caches()
637{
638 bool caches_are_filled = true;
639 if (table_has_data("revisions"))
640 {
641 caches_are_filled = table_has_data("rosters") &&
642 table_has_data("heights");
643 }
644 if (table_has_data("files"))
645 {
646 caches_are_filled = caches_are_filled && table_has_data("file_sizes");
647 }
648
649 E(caches_are_filled, origin::no_fault,
650 F("database '%s' lacks some cached data.\n"
651 "Run '%s db regenerate_caches' to restore use of this database")
652 % filename % prog_name);
653}
654
655static void
656sqlite3_gunzip_fn(sqlite3_context *f, int nargs, sqlite3_value ** args)
657{
658 if (nargs != 1)
659 {
660 sqlite3_result_error(f, "need exactly 1 arg to gunzip()", -1);
661 return;
662 }
663 data unpacked;
664 const char *val = (const char*) sqlite3_value_blob(args[0]);
665 int bytes = sqlite3_value_bytes(args[0]);
666 decode_gzip(gzip<data>(string(val,val+bytes), origin::database), unpacked);
667 sqlite3_result_blob(f, unpacked().c_str(), unpacked().size(), SQLITE_TRANSIENT);
668}
669
670struct sqlite3 *
671database_impl::sql(enum open_mode mode)
672{
673 if (! __sql)
674 {
675 if (type == memory_db)
676 {
677 open();
678
679 sqlite3_exec(__sql, schema_constant, NULL, NULL, NULL);
680 assert_sqlite3_ok(__sql);
681
682 sqlite3_exec(__sql, (FL("PRAGMA user_version = %u;")
683 % mtn_creator_code).str().c_str(), NULL, NULL, NULL);
684 assert_sqlite3_ok(__sql);
685 }
686 else
687 {
688 check_filename();
689 check_db_exists();
690 open();
691
692 if (mode != schema_bypass_mode)
693 {
694 check_sql_schema(__sql, filename);
695
696 if (mode != format_bypass_mode)
697 {
698 check_format();
699
700 if (mode != cache_bypass_mode)
701 check_caches();
702 }
703 }
704 }
705 install_functions();
706 }
707
708 return __sql;
709}
710
711void
712database::initialize()
713{
714 imp->check_filename();
715 imp->check_db_nonexistent();
716 imp->open();
717
718 sqlite3 *sql = imp->__sql;
719
720 sqlite3_exec(sql, schema_constant, NULL, NULL, NULL);
721 assert_sqlite3_ok(sql);
722
723 sqlite3_exec(sql, (FL("PRAGMA user_version = %u;")
724 % mtn_creator_code).str().c_str(), NULL, NULL, NULL);
725 assert_sqlite3_ok(sql);
726
727 // make sure what we wanted is what we got
728 check_sql_schema(sql, imp->filename);
729
730 imp->close();
731}
732
733struct
734dump_request
735{
736 dump_request() : sql(), out() {};
737 struct sqlite3 *sql;
738 ostream *out;
739};
740
741static void
742dump_row(ostream &out, sqlite3_stmt *stmt, string const& table_name)
743{
744 out << FL("INSERT INTO %s VALUES(") % table_name;
745 unsigned n = sqlite3_data_count(stmt);
746 for (unsigned i = 0; i < n; ++i)
747 {
748 if (i != 0)
749 out << ',';
750
751 if (sqlite3_column_type(stmt, i) == SQLITE_BLOB)
752 {
753 out << "X'";
754 const char *val = (const char*) sqlite3_column_blob(stmt, i);
755 int bytes = sqlite3_column_bytes(stmt, i);
756 out << encode_hexenc(string(val,val+bytes), origin::internal);
757 out << '\'';
758 }
759 else
760 {
761 const unsigned char *val = sqlite3_column_text(stmt, i);
762 if (val == NULL)
763 out << "NULL";
764 else
765 {
766 out << '\'';
767 for (const unsigned char *cp = val; *cp; ++cp)
768 {
769 if (*cp == '\'')
770 out << "''";
771 else
772 out << *cp;
773 }
774 out << '\'';
775 }
776 }
777 }
778 out << ");\n";
779}
780
781static int
782dump_table_cb(void *data, int n, char **vals, char **cols)
783{
784 dump_request *dump = reinterpret_cast<dump_request *>(data);
785 I(dump != NULL);
786 I(dump->sql != NULL);
787 I(vals != NULL);
788 I(vals[0] != NULL);
789 I(vals[1] != NULL);
790 I(vals[2] != NULL);
791 I(n == 3);
792 I(string(vals[1]) == "table");
793 *(dump->out) << vals[2] << ";\n";
794 string table_name(vals[0]);
795 string query = "SELECT * FROM " + table_name;
796 sqlite3_stmt *stmt = 0;
797 sqlite3_prepare_v2(dump->sql, query.c_str(), -1, &stmt, NULL);
798 assert_sqlite3_ok(dump->sql);
799
800 int stepresult = SQLITE_DONE;
801 do
802 {
803 stepresult = sqlite3_step(stmt);
804 I(stepresult == SQLITE_DONE || stepresult == SQLITE_ROW);
805 if (stepresult == SQLITE_ROW)
806 dump_row(*(dump->out), stmt, table_name);
807 }
808 while (stepresult == SQLITE_ROW);
809
810 sqlite3_finalize(stmt);
811 assert_sqlite3_ok(dump->sql);
812 return 0;
813}
814
815static int
816dump_index_cb(void *data, int n, char **vals, char **cols)
817{
818 dump_request *dump = reinterpret_cast<dump_request *>(data);
819 I(dump != NULL);
820 I(dump->sql != NULL);
821 I(vals != NULL);
822 I(vals[0] != NULL);
823 I(vals[1] != NULL);
824 I(vals[2] != NULL);
825 I(n == 3);
826 I(string(vals[1]) == "index");
827 *(dump->out) << vals[2] << ";\n";
828 return 0;
829}
830
831static int
832dump_user_version_cb(void *data, int n, char **vals, char **cols)
833{
834 dump_request *dump = reinterpret_cast<dump_request *>(data);
835 I(dump != NULL);
836 I(dump->sql != NULL);
837 I(vals != NULL);
838 I(vals[0] != NULL);
839 I(n == 1);
840 *(dump->out) << "PRAGMA user_version = " << vals[0] << ";\n";
841 return 0;
842}
843
844void
845database::dump(ostream & out)
846{
847 ensure_open_for_maintenance();
848
849 {
850 transaction_guard guard(*this);
851 dump_request req;
852 req.out = &out;
853 req.sql = imp->sql();
854 out << "BEGIN EXCLUSIVE;\n";
855 int res;
856 res = sqlite3_exec(req.sql,
857 "SELECT name, type, sql FROM sqlite_master "
858 "WHERE type='table' AND sql NOT NULL "
859 "AND name not like 'sqlite_stat%' "
860 "ORDER BY name",
861 dump_table_cb, &req, NULL);
862 assert_sqlite3_ok(req.sql);
863 res = sqlite3_exec(req.sql,
864 "SELECT name, type, sql FROM sqlite_master "
865 "WHERE type='index' AND sql NOT NULL "
866 "ORDER BY name",
867 dump_index_cb, &req, NULL);
868 assert_sqlite3_ok(req.sql);
869 res = sqlite3_exec(req.sql,
870 "PRAGMA user_version;",
871 dump_user_version_cb, &req, NULL);
872 assert_sqlite3_ok(req.sql);
873 out << "COMMIT;\n";
874 guard.commit();
875 }
876}
877
878void
879database::load(istream & in)
880{
881 string line;
882 string sql_stmt;
883
884 imp->check_filename();
885 imp->check_db_nonexistent();
886 imp->open();
887
888 sqlite3 * sql = imp->__sql;
889
890 // the page size can only be set before any other commands have been executed
891 sqlite3_exec(sql, "PRAGMA page_size=8192", NULL, NULL, NULL);
892 assert_sqlite3_ok(sql);
893
894 while(in)
895 {
896 getline(in, line, ';');
897 sql_stmt += line + ';';
898
899 if (sqlite3_complete(sql_stmt.c_str()))
900 {
901 sqlite3_exec(sql, sql_stmt.c_str(), NULL, NULL, NULL);
902 assert_sqlite3_ok(sql);
903 sql_stmt.clear();
904 }
905 }
906
907 assert_sqlite3_ok(sql);
908}
909
910
911void
912database::debug(string const & sql, ostream & out)
913{
914 ensure_open_for_maintenance();
915
916 results res;
917 imp->fetch(res, any_cols, any_rows, query(sql));
918 out << '\'' << sql << "' -> " << res.size() << " rows\n\n";
919 for (size_t i = 0; i < res.size(); ++i)
920 {
921 for (size_t j = 0; j < res[i].size(); ++j)
922 {
923 if (j != 0)
924 out << " | ";
925 out << res[i][j];
926 }
927 out << '\n';
928 }
929}
930
931// Subroutine of info(). This compares strings that might either be numbers
932// or error messages surrounded by square brackets. We want the longest
933// number, even if there's an error message that's longer than that.
934static bool longest_number(string a, string b)
935{
936 if(a.length() > 0 && a[0] == '[')
937 return true; // b is longer
938 if(b.length() > 0 && b[0] == '[')
939 return false; // a is longer
940
941 return a.length() < b.length();
942}
943
944// Subroutine of info() and some things it calls.
945// Given an informative_failure which is believed to represent an SQLite
946// error, either return a string version of the error message (if it was an
947// SQLite error) or rethrow the execption (if it wasn't).
948static string
949format_sqlite_error_for_info(recoverable_failure const & e)
950{
951 string err(e.what());
952 string prefix = _("error: ");
953 prefix.append(_("sqlite error: "));
954 if (err.find(prefix) != 0)
955 throw;
956
957 err.replace(0, prefix.length(), "[");
958 string::size_type nl = err.find('\n');
959 if (nl != string::npos)
960 err.erase(nl);
961
962 err.append("]");
963 return err;
964}
965
966// Subroutine of info(). Pretty-print the database's "creator code", which
967// is a 32-bit unsigned number that we interpret as a four-character ASCII
968// string, provided that all four characters are graphic. (On disk, it's
969// stored in the "user version" field of the database.)
970static string
971format_creator_code(u32 code)
972{
973 char buf[5];
974 string result;
975
976 if (code == 0)
977 return _("not set");
978
979 buf[4] = '\0';
980 buf[3] = ((code & 0x000000ff) >> 0);
981 buf[2] = ((code & 0x0000ff00) >> 8);
982 buf[1] = ((code & 0x00ff0000) >> 16);
983 buf[0] = ((code & 0xff000000) >> 24);
984
985 if (isgraph(buf[0]) && isgraph(buf[1]) && isgraph(buf[2]) && isgraph(buf[3]))
986 result = (FL("%s (0x%08x)") % buf % code).str();
987 else
988 result = (FL("0x%08x") % code).str();
989 if (code != mtn_creator_code)
990 result += _(" (not a monotone database)");
991 return result;
992}
993
994
995void
996database::info(ostream & out, bool analyze)
997{
998 // don't check the schema
999 ensure_open_for_maintenance();
1000
1001 // do a dummy query to confirm that the database file is an sqlite3
1002 // database. (this doesn't happen on open() because sqlite postpones the
1003 // actual file open until the first access. we can't piggyback it on the
1004 // query of the user version because there's a bug in sqlite 3.3.10:
1005 // the routine that reads meta-values from the database header does not
1006 // check the file format. reported as sqlite bug #2182.)
1007 sqlite3_exec(imp->__sql, "SELECT 1 FROM sqlite_master LIMIT 0", 0, 0, 0);
1008 assert_sqlite3_ok(imp->__sql);
1009
1010 u32 ccode;
1011 {
1012 results res;
1013 imp->fetch(res, one_col, one_row, query("PRAGMA user_version"));
1014 I(res.size() == 1);
1015 ccode = lexical_cast<u32>(res[0][0]);
1016 }
1017
1018 vector<string> counts;
1019 counts.push_back(imp->count("rosters"));
1020 counts.push_back(imp->count("roster_deltas"));
1021 counts.push_back(imp->count("files"));
1022 counts.push_back(imp->count("file_deltas"));
1023 counts.push_back(imp->count("file_sizes"));
1024 counts.push_back(imp->count("revisions"));
1025 counts.push_back(imp->count("revision_ancestry"));
1026 counts.push_back(imp->count("revision_certs"));
1027
1028 {
1029 results res;
1030 try
1031 {
1032 imp->fetch(res, one_col, any_rows,
1033 query("SELECT node FROM next_roster_node_number"));
1034 if (res.empty())
1035 counts.push_back("0");
1036 else
1037 {
1038 I(res.size() == 1);
1039 u64 n = lexical_cast<u64>(res[0][0]) - 1;
1040 counts.push_back((F("%u") % n).str());
1041 }
1042 }
1043 catch (recoverable_failure const & e)
1044 {
1045 counts.push_back(format_sqlite_error_for_info(e));
1046 }
1047 }
1048
1049 vector<string> bytes;
1050 {
1051 u64 total = 0;
1052 bytes.push_back(imp->space("rosters",
1053 "length(id) + length(checksum) + length(data)",
1054 total));
1055 bytes.push_back(imp->space("roster_deltas",
1056 "length(id) + length(checksum)"
1057 "+ length(base) + length(delta)", total));
1058 bytes.push_back(imp->space("files", "length(id) + length(data)", total));
1059 bytes.push_back(imp->space("file_deltas",
1060 "length(id) + length(base) + length(delta)", total));
1061 bytes.push_back(imp->space("file_sizes",
1062 "length(id) + length(size)", total));
1063 bytes.push_back(imp->space("revisions", "length(id) + length(data)", total));
1064 bytes.push_back(imp->space("revision_ancestry",
1065 "length(parent) + length(child)", total));
1066 bytes.push_back(imp->space("revision_certs",
1067 "length(hash) + length(revision_id) + length(name)"
1068 "+ length(value) + length(keypair_id)"
1069 "+ length(signature)", total));
1070 bytes.push_back(imp->space("heights", "length(revision) + length(height)",
1071 total));
1072 bytes.push_back((F("%u") % total).str());
1073 }
1074
1075 // pad each vector's strings on the left with spaces to make them all the
1076 // same length
1077 {
1078 string::size_type width
1079 = max_element(counts.begin(), counts.end(), longest_number)->length();
1080 for(vector<string>::iterator i = counts.begin(); i != counts.end(); i++)
1081 if (width > i->length() && (*i)[0] != '[')
1082 i->insert(0U, width - i->length(), ' ');
1083
1084 width = max_element(bytes.begin(), bytes.end(), longest_number)->length();
1085 for(vector<string>::iterator i = bytes.begin(); i != bytes.end(); i++)
1086 if (width > i->length() && (*i)[0] != '[')
1087 i->insert(0U, width - i->length(), ' ');
1088 }
1089
1090 i18n_format form =
1091 F("creator code : %s\n"
1092 "schema version : %s\n"
1093 "counts:\n"
1094 " full rosters : %s\n"
1095 " roster deltas : %s\n"
1096 " full files : %s\n"
1097 " file deltas : %s\n"
1098 " file sizes : %s\n"
1099 " revisions : %s\n"
1100 " ancestry edges : %s\n"
1101 " certs : %s\n"
1102 " logical files : %s\n"
1103 "bytes:\n"
1104 " full rosters : %s\n"
1105 " roster deltas : %s\n"
1106 " full files : %s\n"
1107 " file deltas : %s\n"
1108 " file sizes : %s\n"
1109 " revisions : %s\n"
1110 " cached ancestry : %s\n"
1111 " certs : %s\n"
1112 " heights : %s\n"
1113 " total : %s\n"
1114 "database:\n"
1115 " page size : %s\n"
1116 " cache size : %s"
1117 );
1118
1119 form = form % format_creator_code(ccode);
1120 form = form % describe_sql_schema(imp->__sql);
1121
1122 for (vector<string>::iterator i = counts.begin(); i != counts.end(); i++)
1123 form = form % *i;
1124
1125 for (vector<string>::iterator i = bytes.begin(); i != bytes.end(); i++)
1126 form = form % *i;
1127
1128 form = form % imp->page_size();
1129 form = form % imp->cache_size();
1130
1131 out << form.str() << '\n'; // final newline is kept out of the translation
1132
1133 // the following analyzation is only done for --verbose info
1134 if (!analyze)
1135 return;
1136
1137
1138 typedef map<revision_id, date_t> rev_date;
1139 rev_date rd;
1140 vector<cert> certs;
1141
1142 L(FL("fetching revision dates"));
1143 imp->get_certs(date_cert_name, certs, "revision_certs");
1144
1145 L(FL("analyzing revision dates"));
1146 rev_date::iterator d;
1147 for (vector<cert>::iterator i = certs.begin(); i != certs.end(); ++i)
1148 {
1149 date_t cert_date;
1150 try
1151 {
1152 cert_date = date_t(i->value());
1153 }
1154 catch (recoverable_failure & e)
1155 {
1156 // simply skip dates we cannot parse
1157 W(F("invalid date '%s' for revision %s; skipped")
1158 % i->value() % i->ident);
1159 }
1160
1161 if (cert_date.valid())
1162 {
1163 if ((d = rd.find(i->ident)) == rd.end())
1164 rd.insert(make_pair(i->ident, cert_date));
1165 else
1166 {
1167 if (d->second > cert_date)
1168 d->second = cert_date;
1169 }
1170 }
1171 }
1172
1173 L(FL("fetching ancestry map"));
1174 typedef multimap<revision_id, revision_id>::const_iterator gi;
1175 rev_ancestry_map graph;
1176 get_forward_ancestry(graph);
1177
1178 L(FL("checking timestamps differences of related revisions"));
1179 int correct = 0,
1180 equal = 0,
1181 incorrect = 0,
1182 root_anc = 0,
1183 missing = 0;
1184
1185 vector<s64> diffs;
1186
1187 for (gi i = graph.begin(); i != graph.end(); ++i)
1188 {
1189 revision_id anc_rid = i->first,
1190 desc_rid = i->second;
1191
1192 if (null_id(anc_rid))
1193 {
1194 root_anc++;
1195 continue;
1196 }
1197 I(!null_id(desc_rid));
1198
1199 date_t anc_date,
1200 desc_date;
1201
1202 map<revision_id, date_t>::iterator j;
1203 if ((j = rd.find(anc_rid)) != rd.end())
1204 anc_date = j->second;
1205
1206 if ((j = rd.find(desc_rid)) != rd.end())
1207 desc_date = j->second;
1208
1209 if (anc_date.valid() && desc_date.valid())
1210 {
1211 // we only need seconds precision here
1212 s64 diff = (desc_date - anc_date) / 1000;
1213 diffs.push_back(diff);
1214
1215 if (anc_date < desc_date)
1216 correct++;
1217 else if (anc_date == desc_date)
1218 equal++;
1219 else
1220 {
1221 L(FL(" rev %s -> rev %s") % anc_rid % desc_rid);
1222 L(FL(" but date %s ! -> %s")
1223 % anc_date.as_iso_8601_extended()
1224 % desc_date.as_iso_8601_extended());
1225 L(FL(" (difference: %d seconds)")
1226 % (anc_date - desc_date));
1227 incorrect++;
1228 }
1229 }
1230 else
1231 missing++;
1232 }
1233
1234 // no information to provide in this case
1235 if (diffs.size() == 0)
1236 return;
1237
1238 form =
1239 F("timestamp correctness between revisions:\n"
1240 " correct dates : %s edges\n"
1241 " equal dates : %s edges\n"
1242 " incorrect dates : %s edges\n"
1243 " based on root : %s edges\n"
1244 " missing dates : %s edges\n"
1245 "\n"
1246 "timestamp differences between revisions:\n"
1247 " mean : %d sec\n"
1248 " min : %d sec\n"
1249 " max : %d sec\n"
1250 "\n"
1251 " 1st percentile : %s sec\n"
1252 " 5th percentile : %s sec\n"
1253 " 10th percentile : %s sec\n"
1254 " 25th percentile : %s sec\n"
1255 " 50th percentile : %s sec\n"
1256 " 75th percentile : %s sec\n"
1257 " 90th percentile : %s sec\n"
1258 " 95th percentile : %s sec\n"
1259 " 99th percentile : %s sec\n"
1260 );
1261
1262 form = form % correct % equal % incorrect % root_anc % missing;
1263
1264 // sort, so that we can get percentile values
1265 sort(diffs.begin(), diffs.end());
1266
1267 // calculate mean time difference, output that, min and max
1268 s64 mean = accumulate(diffs.begin(), diffs.end(), 0);
1269 mean /= diffs.size();
1270 s64 median = *(diffs.begin() + diffs.size()/2);
1271 form = form % mean % *diffs.begin() % *diffs.rbegin()
1272 % *(diffs.begin() + int(diffs.size() * 0.01))
1273 % *(diffs.begin() + int(diffs.size() * 0.05))
1274 % *(diffs.begin() + int(diffs.size() * 0.10))
1275 % *(diffs.begin() + int(diffs.size() * 0.25))
1276 % *(diffs.begin() + int(diffs.size() * 0.50))
1277 % *(diffs.begin() + int(diffs.size() * 0.75))
1278 % *(diffs.begin() + int(diffs.size() * 0.90))
1279 % *(diffs.begin() + int(diffs.size() * 0.95))
1280 % *(diffs.begin() + int(diffs.size() * 0.99));
1281
1282 // output the string, with some newlines out of translation
1283 out << '\n' << '\n' << form.str() << '\n';
1284}
1285
1286void
1287database::version(ostream & out)
1288{
1289 ensure_open_for_maintenance();
1290 out << (F("database schema version: %s")
1291 % describe_sql_schema(imp->__sql)).str()
1292 << '\n';
1293}
1294
1295void
1296database::migrate(key_store & keys, migration_status & mstat)
1297{
1298 ensure_open_for_maintenance();
1299 mstat = migrate_sql_schema(imp->__sql, keys, get_filename());
1300}
1301
1302void
1303database::test_migration_step(key_store & keys, string const & schema)
1304{
1305 ensure_open_for_maintenance();
1306 ::test_migration_step(imp->__sql, keys, get_filename(), schema);
1307}
1308
1309void
1310database::fix_bad_certs(bool drop_not_fixable)
1311{
1312 vector<key_id> all_keys;
1313 get_key_ids(all_keys);
1314
1315 P(F("loading certs"));
1316 vector<pair<id, cert> > all_certs;
1317 {
1318 results res;
1319 query q("SELECT revision_id, name, value, keypair_id, signature, hash FROM revision_certs");
1320 imp->fetch(res, 6, any_rows, q);
1321 imp->results_to_certs(res, all_certs);
1322 }
1323
1324 P(F("checking"));
1325
1326 ticker tick_checked(_("checked"), "c", 25);
1327 ticker tick_bad(_("bad"), "b", 1);
1328 ticker tick_fixed(_("fixed"), "f", 1);
1329 shared_ptr<ticker> tick_dropped;
1330 if (drop_not_fixable)
1331 tick_dropped.reset(new ticker(_("dropped"), "d", 1));
1332 tick_checked.set_total(all_certs.size());
1333
1334 int num_bad(0), num_fixed(0), num_dropped(0);
1335
1336 for (vector<pair<id, cert> >::const_iterator cert_iter = all_certs.begin();
1337 cert_iter != all_certs.end(); ++cert_iter)
1338 {
1339 cert const & c(cert_iter->second);
1340 id const & certid(cert_iter->first);
1341 cert_status status = check_cert(c);
1342 ++tick_checked;
1343 if (status == cert_bad)
1344 {
1345 ++tick_bad;
1346 ++num_bad;
1347 bool fixed = false;
1348 string signable;
1349 c.signable_text(signable);
1350 for (vector<key_id>::const_iterator key_iter = all_keys.begin();
1351 key_iter != all_keys.end(); ++key_iter)
1352 {
1353 key_id const & keyid(*key_iter);
1354 if (check_signature(keyid, signable, c.sig) == cert_ok)
1355 {
1356 key_name candidate_name;
1357 rsa_pub_key junk;
1358 get_pubkey(keyid, candidate_name, junk);
1359 id chk_id;
1360 c.hash_code(candidate_name, chk_id);
1361 if (chk_id == certid)
1362 {
1363 imp->execute(query("UPDATE revision_certs SET keypair_id = ? WHERE hash = ?")
1364 % blob(keyid.inner()()) % blob(certid()));
1365 ++tick_fixed;
1366 ++num_fixed;
1367 fixed = true;
1368 break;
1369 }
1370 }
1371 }
1372 if (!fixed)
1373 {
1374 if (drop_not_fixable)
1375 {
1376 imp->execute(query("DELETE FROM revision_certs WHERE hash = ?")
1377 % blob(certid()));
1378 ++(*tick_dropped);
1379 ++num_dropped;
1380 }
1381 }
1382 }
1383 }
1384 if (drop_not_fixable)
1385 {
1386 P(F("checked %d certs, found %d bad, fixed %d, dropped %d")
1387 % all_certs.size() % num_bad % num_fixed % num_dropped);
1388 }
1389 else
1390 {
1391 P(F("checked %d certs, found %d bad, fixed %d")
1392 % all_certs.size() % num_bad % num_fixed);
1393 }
1394}
1395
1396void
1397database::ensure_open()
1398{
1399 imp->sql();
1400}
1401
1402void
1403database::ensure_open_for_format_changes()
1404{
1405 imp->sql(format_bypass_mode);
1406}
1407
1408void
1409database::ensure_open_for_cache_reset()
1410{
1411 imp->sql(cache_bypass_mode);
1412}
1413
1414void
1415database::ensure_open_for_maintenance()
1416{
1417 imp->sql(schema_bypass_mode);
1418}
1419
1420void
1421database_impl::execute(query const & query)
1422{
1423 results res;
1424 fetch(res, 0, 0, query);
1425}
1426
1427void
1428database_impl::fetch(results & res,
1429 int const want_cols,
1430 int const want_rows,
1431 query const & query)
1432{
1433 int nrow;
1434 int ncol;
1435 int rescode;
1436
1437 res.clear();
1438 res.resize(0);
1439
1440 map<string, statement>::iterator i = statement_cache.find(query.sql_cmd);
1441 if (i == statement_cache.end())
1442 {
1443 statement_cache.insert(make_pair(query.sql_cmd, statement()));
1444 i = statement_cache.find(query.sql_cmd);
1445 I(i != statement_cache.end());
1446
1447 const char * tail;
1448 sqlite3_prepare_v2(sql(), query.sql_cmd.c_str(), -1, i->second.stmt.paddr(), &tail);
1449 assert_sqlite3_ok(sql());
1450 L(FL("prepared statement %s") % query.sql_cmd);
1451
1452 // no support for multiple statements here
1453 E(*tail == 0, origin::internal,
1454 F("multiple statements in query: %s") % query.sql_cmd);
1455 }
1456
1457 ncol = sqlite3_column_count(i->second.stmt());
1458
1459 E(want_cols == any_cols || want_cols == ncol, origin::database,
1460 F("wanted %d columns got %d in query: %s") % want_cols % ncol % query.sql_cmd);
1461
1462 // bind parameters for this execution
1463
1464 int params = sqlite3_bind_parameter_count(i->second.stmt());
1465
1466 // Ensure that exactly the right number of parameters were given
1467 I(params == int(query.args.size()));
1468
1469 L(FL("binding %d parameters for %s") % params % query.sql_cmd);
1470
1471 for (int param = 1; param <= params; param++)
1472 {
1473 // profiling finds this logging to be quite expensive
1474 if (global_sanity.debug_p())
1475 {
1476 string prefix;
1477 string log;
1478
1479 if (query.args[param-1].type == query_param::blob)
1480 {
1481 prefix = "x";
1482 log = encode_hexenc(query.args[param-1].string_data, origin::internal);
1483 }
1484 else if (query.args[param-1].type == query_param::int64)
1485 {
1486 log = lexical_cast<string>(query.args[param-1].int_data);
1487 }
1488 else
1489 {
1490 log = query.args[param-1].string_data;
1491 }
1492
1493 if (log.size() > constants::db_log_line_sz)
1494 log = log.substr(0, constants::db_log_line_sz - 2) + "..";
1495
1496 L(FL("binding %d with value '%s'") % param % log);
1497 }
1498
1499 switch (idx(query.args, param - 1).type)
1500 {
1501 case query_param::text:
1502 sqlite3_bind_text(i->second.stmt(), param,
1503 idx(query.args, param - 1).string_data.c_str(), -1,
1504 SQLITE_STATIC);
1505 break;
1506 case query_param::blob:
1507 {
1508 string const & data = idx(query.args, param - 1).string_data;
1509 sqlite3_bind_blob(i->second.stmt(), param,
1510 data.data(), data.size(),
1511 SQLITE_STATIC);
1512 }
1513 break;
1514 case query_param::int64:
1515 {
1516 u64 data = idx(query.args, param - 1).int_data;
1517 sqlite3_bind_int64(i->second.stmt(), param, data);
1518 }
1519 break;
1520 default:
1521 I(false);
1522 }
1523
1524 assert_sqlite3_ok(sql());
1525 }
1526
1527 // execute and process results
1528
1529 nrow = 0;
1530 for (rescode = sqlite3_step(i->second.stmt()); rescode == SQLITE_ROW;
1531 rescode = sqlite3_step(i->second.stmt()))
1532 {
1533 vector<string> row;
1534 for (int col = 0; col < ncol; col++)
1535 {
1536 // We never store NULLs, so we should never see one.
1537 int const datatype = sqlite3_column_type(i->second.stmt(), col);
1538 E(datatype != SQLITE_NULL, origin::database,
1539 F("null result in query: %s") % query.sql_cmd);
1540 const char * value = (const char*)sqlite3_column_blob(i->second.stmt(), col);
1541 int bytes = sqlite3_column_bytes(i->second.stmt(), col);
1542 if (value) {
1543 row.push_back(string(value, value + bytes));
1544 } else {
1545 // sqlite3_column_blob() returns null for zero-length
1546 I(bytes == 0);
1547 row.push_back(string());
1548 }
1549 }
1550 res.push_back(row);
1551 }
1552
1553 if (rescode != SQLITE_DONE)
1554 assert_sqlite3_ok(sql());
1555
1556 sqlite3_reset(i->second.stmt());
1557 assert_sqlite3_ok(sql());
1558
1559 nrow = res.size();
1560
1561 i->second.count++;
1562
1563 E(want_rows == any_rows || want_rows == nrow,
1564 origin::database,
1565 F("wanted %d rows got %d in query: %s") % want_rows % nrow % query.sql_cmd);
1566}
1567
1568bool
1569database_impl::table_has_entry(id const & key,
1570 std::string const & column,
1571 std::string const & table)
1572{
1573 results res;
1574 query q("SELECT 1 FROM " + table + " WHERE " + column + " = ? LIMIT 1");
1575 fetch(res, one_col, any_rows, q % blob(key()));
1576 return !res.empty();
1577}
1578
1579// general application-level logic
1580
1581void
1582database_impl::begin_transaction(bool exclusive)
1583{
1584 if (transaction_level == 0)
1585 {
1586 I(delayed_files.empty());
1587 I(roster_cache.all_clean());
1588 if (exclusive)
1589 execute(query("BEGIN EXCLUSIVE"));
1590 else
1591 execute(query("BEGIN DEFERRED"));
1592 transaction_exclusive = exclusive;
1593 }
1594 else
1595 {
1596 // You can't start an exclusive transaction within a non-exclusive
1597 // transaction
1598 I(!exclusive || transaction_exclusive);
1599 }
1600 transaction_level++;
1601}
1602
1603
1604static size_t
1605size_delayed_file(file_id const & id, file_data const & dat)
1606{
1607 return id.inner()().size() + dat.inner()().size();
1608}
1609
1610bool
1611database_impl::have_delayed_file(file_id const & id)
1612{
1613 return delayed_files.find(id) != delayed_files.end();
1614}
1615
1616void
1617database_impl::load_delayed_file(file_id const & id, file_data & dat)
1618{
1619 dat = safe_get(delayed_files, id);
1620}
1621
1622// precondition: have_delayed_file(an_id) == true
1623void
1624database_impl::cancel_delayed_file(file_id const & an_id)
1625{
1626 file_data const & dat = safe_get(delayed_files, an_id);
1627 size_t cancel_size = size_delayed_file(an_id, dat);
1628 I(cancel_size <= delayed_writes_size);
1629 delayed_writes_size -= cancel_size;
1630
1631 safe_erase(delayed_files, an_id);
1632}
1633
1634void
1635database_impl::drop_or_cancel_file(file_id const & id)
1636{
1637 if (have_delayed_file(id))
1638 cancel_delayed_file(id);
1639 else
1640 drop(id.inner(), "files");
1641}
1642
1643void
1644database_impl::schedule_delayed_file(file_id const & an_id,
1645 file_data const & dat)
1646{
1647 if (!have_delayed_file(an_id))
1648 {
1649 safe_insert(delayed_files, make_pair(an_id, dat));
1650 delayed_writes_size += size_delayed_file(an_id, dat);
1651 }
1652 if (delayed_writes_size > constants::db_max_delayed_file_bytes)
1653 flush_delayed_writes();
1654}
1655
1656void
1657database_impl::flush_delayed_writes()
1658{
1659 for (map<file_id, file_data>::const_iterator i = delayed_files.begin();
1660 i != delayed_files.end(); ++i)
1661 write_delayed_file(i->first, i->second);
1662 clear_delayed_writes();
1663}
1664
1665void
1666database_impl::clear_delayed_writes()
1667{
1668 delayed_files.clear();
1669 delayed_writes_size = 0;
1670}
1671
1672void
1673database_impl::roster_writeback_manager::writeout(revision_id const & id,
1674 cached_roster const & cr)
1675{
1676 I(cr.first);
1677 I(cr.second);
1678 imp.write_delayed_roster(id, *(cr.first), *(cr.second));
1679}
1680
1681void
1682database_impl::commit_transaction()
1683{
1684 if (transaction_level == 1)
1685 {
1686 flush_delayed_writes();
1687 roster_cache.clean_all();
1688 execute(query("COMMIT"));
1689 }
1690 transaction_level--;
1691}
1692
1693void
1694database_impl::rollback_transaction()
1695{
1696 if (transaction_level == 1)
1697 {
1698 clear_delayed_writes();
1699 roster_cache.clear_and_drop_writes();
1700 execute(query("ROLLBACK"));
1701 }
1702 transaction_level--;
1703}
1704
1705
1706bool
1707database_impl::file_or_manifest_base_exists(id const & ident,
1708 string const & table)
1709{
1710 // just check for a delayed file, since there are no delayed manifests
1711 if (have_delayed_file(file_id(ident)))
1712 return true;
1713 return table_has_entry(ident, "id", table);
1714}
1715
1716bool
1717database::file_or_manifest_base_exists(file_id const & ident,
1718 string const & table)
1719{
1720 return imp->file_or_manifest_base_exists(ident.inner(), table);
1721}
1722
1723// returns true if we are currently storing (or planning to store) a
1724// full-text for 'ident'
1725bool
1726database_impl::roster_base_stored(revision_id const & ident)
1727{
1728 if (roster_cache.exists(ident) && roster_cache.is_dirty(ident))
1729 return true;
1730 return table_has_entry(ident.inner(), "id", "rosters");
1731}
1732
1733// returns true if we currently have a full-text for 'ident' available
1734// (possibly cached). Warning: the results of this method are invalidated
1735// by calling roster_cache.insert_{clean,dirty}, because they can trigger
1736// cache cleaning.
1737bool
1738database_impl::roster_base_available(revision_id const & ident)
1739{
1740 if (roster_cache.exists(ident))
1741 return true;
1742 return table_has_entry(ident.inner(), "id", "rosters");
1743}
1744
1745bool
1746database::delta_exists(id const & ident,
1747 string const & table)
1748{
1749 return imp->table_has_entry(ident, "id", table);
1750}
1751
1752bool
1753database_impl::delta_exists(file_id const & ident,
1754 file_id const & base,
1755 string const & table)
1756{
1757 results res;
1758 query q("SELECT 1 FROM " + table + " WHERE id = ? and base = ? LIMIT 1");
1759 fetch(res, one_col, any_rows,
1760 q % blob(ident.inner()()) % blob(base.inner()()));
1761 return !res.empty();
1762}
1763
1764string
1765database_impl::count(string const & table)
1766{
1767 try
1768 {
1769 results res;
1770 query q("SELECT COUNT(*) FROM " + table);
1771 fetch(res, one_col, one_row, q);
1772 return (F("%u") % lexical_cast<u64>(res[0][0])).str();
1773 }
1774 catch (recoverable_failure const & e)
1775 {
1776 return format_sqlite_error_for_info(e);
1777 }
1778
1779}
1780
1781string
1782database_impl::space(string const & table, string const & rowspace, u64 & total)
1783{
1784 try
1785 {
1786 results res;
1787 // SUM({empty set}) is NULL; TOTAL({empty set}) is 0.0
1788 query q("SELECT TOTAL(" + rowspace + ") FROM " + table);
1789 fetch(res, one_col, one_row, q);
1790 u64 bytes = static_cast<u64>(lexical_cast<double>(res[0][0]));
1791 total += bytes;
1792 return (F("%u") % bytes).str();
1793 }
1794 catch (recoverable_failure & e)
1795 {
1796 return format_sqlite_error_for_info(e);
1797 }
1798}
1799
1800unsigned int
1801database_impl::page_size()
1802{
1803 results res;
1804 query q("PRAGMA page_size");
1805 fetch(res, one_col, one_row, q);
1806 return lexical_cast<unsigned int>(res[0][0]);
1807}
1808
1809unsigned int
1810database_impl::cache_size()
1811{
1812 // This returns the persistent (default) cache size. It's possible to
1813 // override this setting transiently at runtime by setting PRAGMA
1814 // cache_size.
1815 results res;
1816 query q("PRAGMA default_cache_size");
1817 fetch(res, one_col, one_row, q);
1818 return lexical_cast<unsigned int>(res[0][0]);
1819}
1820
1821void
1822database_impl::get_ids(string const & table, set<id> & ids)
1823{
1824 results res;
1825 query q("SELECT id FROM " + table);
1826 fetch(res, one_col, any_rows, q);
1827
1828 for (size_t i = 0; i < res.size(); ++i)
1829 {
1830 ids.insert(id(res[i][0], origin::database));
1831 }
1832}
1833
1834// for files and legacy manifest support
1835void
1836database_impl::get_file_or_manifest_base_unchecked(id const & ident,
1837 data & dat,
1838 string const & table)
1839{
1840 if (have_delayed_file(file_id(ident)))
1841 {
1842 file_data tmp;
1843 load_delayed_file(file_id(ident), tmp);
1844 dat = tmp.inner();
1845 return;
1846 }
1847
1848 results res;
1849 query q("SELECT data FROM " + table + " WHERE id = ?");
1850 fetch(res, one_col, one_row, q % blob(ident()));
1851
1852 gzip<data> rdata(res[0][0], origin::database);
1853 data rdata_unpacked;
1854 decode_gzip(rdata,rdata_unpacked);
1855
1856 dat = rdata_unpacked;
1857}
1858
1859// for files and legacy manifest support
1860void
1861database_impl::get_file_or_manifest_delta_unchecked(id const & ident,
1862 id const & base,
1863 delta & del,
1864 string const & table)
1865{
1866 I(ident() != "");
1867 I(base() != "");
1868 results res;
1869 query q("SELECT delta FROM " + table + " WHERE id = ? AND base = ?");
1870 fetch(res, one_col, one_row,
1871 q % blob(ident()) % blob(base()));
1872
1873 gzip<delta> del_packed(res[0][0], origin::database);
1874 decode_gzip(del_packed, del);
1875}
1876
1877void
1878database_impl::get_roster_base(revision_id const & ident,
1879 roster_t & roster, marking_map & marking)
1880{
1881 if (roster_cache.exists(ident))
1882 {
1883 cached_roster cr;
1884 roster_cache.fetch(ident, cr);
1885 I(cr.first);
1886 roster = *(cr.first);
1887 I(cr.second);
1888 marking = *(cr.second);
1889 return;
1890 }
1891 results res;
1892 query q("SELECT checksum, data FROM rosters WHERE id = ?");
1893 fetch(res, 2, one_row, q % blob(ident.inner()()));
1894
1895 id checksum(res[0][0], origin::database);
1896 id calculated;
1897 calculate_ident(data(res[0][1], origin::database), calculated);
1898 E(calculated == checksum, origin::database,
1899 F("roster does not match hash"));
1900
1901 gzip<data> dat_packed(res[0][1], origin::database);
1902 data dat;
1903 decode_gzip(dat_packed, dat);
1904 read_roster_and_marking(roster_data(dat), roster, marking);
1905}
1906
1907void
1908database_impl::get_roster_delta(id const & ident,
1909 id const & base,
1910 roster<delta> & del)
1911{
1912 results res;
1913 query q("SELECT checksum, delta FROM roster_deltas WHERE id = ? AND base = ?");
1914 fetch(res, 2, one_row, q % blob(ident()) % blob(base()));
1915
1916 id checksum(res[0][0], origin::database);
1917 id calculated;
1918 calculate_ident(data(res[0][1], origin::database), calculated);
1919 E(calculated == checksum, origin::database,
1920 F("roster_delta does not match hash"));
1921
1922 gzip<delta> del_packed(res[0][1], origin::database);
1923 delta tmp;
1924 decode_gzip(del_packed, tmp);
1925 del = roster<delta>(tmp);
1926}
1927
1928void
1929database_impl::write_delayed_file(file_id const & ident,
1930 file_data const & dat)
1931{
1932 gzip<data> dat_packed;
1933 encode_gzip(dat.inner(), dat_packed);
1934
1935 // ident is a hash, which we should check
1936 I(!null_id(ident));
1937 file_id tid;
1938 calculate_ident(dat, tid);
1939 MM(ident);
1940 MM(tid);
1941 I(tid == ident);
1942 // and then write things to the db
1943 query q("INSERT INTO files (id, data) VALUES (?, ?)");
1944 execute(q % blob(ident.inner()()) % blob(dat_packed()));
1945}
1946
1947void
1948database_impl::write_delayed_roster(revision_id const & ident,
1949 roster_t const & roster,
1950 marking_map const & marking)
1951{
1952 roster_data dat;
1953 write_roster_and_marking(roster, marking, dat);
1954 gzip<data> dat_packed;
1955 encode_gzip(dat.inner(), dat_packed);
1956
1957 // ident is a number, and we should calculate a checksum on what
1958 // we write
1959 id checksum;
1960 calculate_ident(typecast_vocab<data>(dat_packed), checksum);
1961
1962 // and then write it
1963 execute(query("INSERT INTO rosters (id, checksum, data) VALUES (?, ?, ?)")
1964 % blob(ident.inner()())
1965 % blob(checksum())
1966 % blob(dat_packed()));
1967}
1968
1969
1970void
1971database::put_file_delta(file_id const & ident,
1972 file_id const & base,
1973 file_delta const & del)
1974{
1975 I(!null_id(ident));
1976 I(!null_id(base));
1977
1978 gzip<delta> del_packed;
1979 encode_gzip(del.inner(), del_packed);
1980
1981 imp->execute(query("INSERT INTO file_deltas (id, base, delta) VALUES (?, ?, ?)")
1982 % blob(ident.inner()())
1983 % blob(base.inner()())
1984 % blob(del_packed()));
1985}
1986
1987void
1988database_impl::put_file_size(file_id const & ident,
1989 file_data const & data)
1990{
1991 I(!null_id(ident));
1992 file_size size = data.inner()().size();
1993 // really identical files should be rather rare, so the cost of checking
1994 // whether an entry exists _everytime_ by hand should be higher than just
1995 // replacing any (possibly existing) entry. and since each identical file
1996 // should also have an identical size, we're done here
1997 query q("INSERT OR REPLACE INTO file_sizes(id, size) VALUES (?, ?)");
1998 execute(q % blob(ident.inner()()) % int64(size));
1999}
2000
2001void
2002database_impl::put_roster_delta(revision_id const & ident,
2003 revision_id const & base,
2004 roster_delta const & del)
2005{
2006 gzip<delta> del_packed;
2007 encode_gzip(del.inner(), del_packed);
2008
2009 id checksum;
2010 calculate_ident(typecast_vocab<data>(del_packed), checksum);
2011
2012 query q("INSERT INTO roster_deltas (id, base, checksum, delta) VALUES (?, ?, ?, ?)");
2013 execute(q
2014 % blob(ident.inner()())
2015 % blob(base.inner()())
2016 % blob(checksum())
2017 % blob(del_packed()));
2018}
2019
2020struct file_and_manifest_reconstruction_graph : public reconstruction_graph
2021{
2022 database_impl & imp;
2023 string const & data_table;
2024 string const & delta_table;
2025
2026 file_and_manifest_reconstruction_graph(database_impl & imp,
2027 string const & data_table,
2028 string const & delta_table)
2029 : imp(imp), data_table(data_table), delta_table(delta_table)
2030 {}
2031 virtual bool is_base(id const & node) const
2032 {
2033 return imp.vcache.exists(node)
2034 || imp.file_or_manifest_base_exists(node, data_table);
2035 }
2036 virtual void get_next(id const & from, set<id> & next) const
2037 {
2038 next.clear();
2039 results res;
2040 query q("SELECT base FROM " + delta_table + " WHERE id = ?");
2041 imp.fetch(res, one_col, any_rows, q % blob(from()));
2042 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
2043 next.insert(id((*i)[0], origin::database));
2044 }
2045};
2046
2047// used for files and legacy manifest migration
2048void
2049database_impl::get_version(id const & ident,
2050 data & dat,
2051 string const & data_table,
2052 string const & delta_table)
2053{
2054 I(ident() != "");
2055
2056 reconstruction_path selected_path;
2057 {
2058 file_and_manifest_reconstruction_graph graph(*this, data_table, delta_table);
2059 get_reconstruction_path(ident, graph, selected_path);
2060 }
2061
2062 I(!selected_path.empty());
2063
2064 id curr = selected_path.back();
2065 selected_path.pop_back();
2066 data begin;
2067
2068 if (vcache.exists(curr))
2069 I(vcache.fetch(curr, begin));
2070 else
2071 get_file_or_manifest_base_unchecked(curr, begin, data_table);
2072
2073 shared_ptr<delta_applicator> appl = new_piecewise_applicator();
2074 appl->begin(begin());
2075
2076 for (reconstruction_path::reverse_iterator i = selected_path.rbegin();
2077 i != selected_path.rend(); ++i)
2078 {
2079 id const nxt = id(*i);
2080
2081 if (!vcache.exists(curr))
2082 {
2083 string tmp;
2084 appl->finish(tmp);
2085 vcache.insert_clean(curr, data(tmp, origin::database));
2086 }
2087
2088 if (global_sanity.debug_p())
2089 L(FL("following delta %s -> %s") % curr % nxt);
2090 delta del;
2091 get_file_or_manifest_delta_unchecked(nxt, curr, del, delta_table);
2092 apply_delta(appl, del());
2093
2094 appl->next();
2095 curr = nxt;
2096 }
2097
2098 string tmp;
2099 appl->finish(tmp);
2100 dat = data(tmp, origin::database);
2101
2102 id final;
2103 calculate_ident(dat, final);
2104 E(final == ident, origin::database,
2105 F("delta-reconstructed '%s' item does not match hash")
2106 % data_table);
2107
2108 if (!vcache.exists(ident))
2109 vcache.insert_clean(ident, dat);
2110}
2111
2112struct roster_reconstruction_graph : public reconstruction_graph
2113{
2114 database_impl & imp;
2115 roster_reconstruction_graph(database_impl & imp) : imp(imp) {}
2116 virtual bool is_base(id const & node) const
2117 {
2118 return imp.roster_base_available(revision_id(node));
2119 }
2120 virtual void get_next(id const & from, set<id> & next) const
2121 {
2122 next.clear();
2123 results res;
2124 query q("SELECT base FROM roster_deltas WHERE id = ?");
2125 imp.fetch(res, one_col, any_rows, q % blob(from()));
2126 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
2127 next.insert(id((*i)[0], origin::database));
2128 }
2129};
2130
2131struct database_impl::extractor
2132{
2133 virtual bool look_at_delta(roster_delta const & del) = 0;
2134 virtual void look_at_roster(roster_t const & roster, marking_map const & mm) = 0;
2135 virtual ~extractor() {};
2136};
2137
2138struct database_impl::markings_extractor : public database_impl::extractor
2139{
2140private:
2141 node_id const & nid;
2142 const_marking_t & markings;
2143
2144public:
2145 markings_extractor(node_id const & _nid, const_marking_t & _markings) :
2146 nid(_nid), markings(_markings) {} ;
2147
2148 bool look_at_delta(roster_delta const & del)
2149 {
2150 return try_get_markings_from_roster_delta(del, nid, markings);
2151 }
2152
2153 void look_at_roster(roster_t const & roster, marking_map const & mm)
2154 {
2155 markings = mm.get_marking(nid);
2156 }
2157};
2158
2159struct database_impl::file_content_extractor : database_impl::extractor
2160{
2161private:
2162 node_id const & nid;
2163 file_id & content;
2164
2165public:
2166 file_content_extractor(node_id const & _nid, file_id & _content) :
2167 nid(_nid), content(_content) {} ;
2168
2169 bool look_at_delta(roster_delta const & del)
2170 {
2171 return try_get_content_from_roster_delta(del, nid, content);
2172 }
2173
2174 void look_at_roster(roster_t const & roster, marking_map const & mm)
2175 {
2176 if (roster.has_node(nid))
2177 content = downcast_to_file_t(roster.get_node(nid))->content;
2178 else
2179 content = file_id();
2180 }
2181};
2182
2183void
2184database_impl::extract_from_deltas(revision_id const & ident, extractor & x)
2185{
2186 reconstruction_path selected_path;
2187 {
2188 roster_reconstruction_graph graph(*this);
2189 {
2190 // we look at the nearest delta(s) first, without constructing the
2191 // whole path, as that would be a rather expensive operation.
2192 //
2193 // the reason why this strategy is worth the effort is, that in most
2194 // cases we are looking at the parent of a (content-)marked node, thus
2195 // the information we are for is right there in the delta leading to
2196 // this node.
2197 //
2198 // recording the deltas visited here in a set as to avoid inspecting
2199 // them later seems to be of little value, as it imposes a cost here,
2200 // but can seldom be exploited.
2201 set<id> deltas;
2202 graph.get_next(ident.inner(), deltas);
2203 for (set<id>::const_iterator i = deltas.begin();
2204 i != deltas.end(); ++i)
2205 {
2206 roster_delta del;
2207 get_roster_delta(ident.inner(), *i, del);
2208 bool found = x.look_at_delta(del);
2209 if (found)
2210 return;
2211 }
2212 }
2213 get_reconstruction_path(ident.inner(), graph, selected_path);
2214 }
2215
2216 int path_length(selected_path.size());
2217 int i(0);
2218 id target_rev;
2219
2220 for (reconstruction_path::const_iterator p = selected_path.begin();
2221 p != selected_path.end(); ++p)
2222 {
2223 if (i > 0)
2224 {
2225 roster_delta del;
2226 get_roster_delta(target_rev, id(*p), del);
2227 bool found = x.look_at_delta(del);
2228 if (found)
2229 return;
2230 }
2231 if (i == path_length-1)
2232 {
2233 // last iteration, we have reached a roster base
2234 roster_t roster;
2235 marking_map mm;
2236 get_roster_base(revision_id(*p), roster, mm);
2237 x.look_at_roster(roster, mm);
2238 return;
2239 }
2240 target_rev = id(*p);
2241 ++i;
2242 }
2243}
2244
2245void
2246database::get_markings(revision_id const & id,
2247 node_id const & nid,
2248 const_marking_t & markings)
2249{
2250 database_impl::markings_extractor x(nid, markings);
2251 imp->extract_from_deltas(id, x);
2252}
2253
2254void
2255database::get_file_content(revision_id const & id,
2256 node_id const & nid,
2257 file_id & content)
2258{
2259 // the imaginary root revision doesn't have any file.
2260 if (null_id(id))
2261 {
2262 content = file_id();
2263 return;
2264 }
2265 database_impl::file_content_extractor x(nid, content);
2266 imp->extract_from_deltas(id, x);
2267}
2268
2269void
2270database::get_roster_version(revision_id const & ros_id,
2271 cached_roster & cr)
2272{
2273 // if we already have it, exit early
2274 if (imp->roster_cache.exists(ros_id))
2275 {
2276 imp->roster_cache.fetch(ros_id, cr);
2277 return;
2278 }
2279
2280 reconstruction_path selected_path;
2281 {
2282 roster_reconstruction_graph graph(*imp);
2283 get_reconstruction_path(ros_id.inner(), graph, selected_path);
2284 }
2285
2286 id curr(selected_path.back());
2287 selected_path.pop_back();
2288 // we know that this isn't already in the cache (because of the early exit
2289 // above), so we should create new objects and spend time filling them in.
2290 shared_ptr<roster_t> roster(new roster_t);
2291 shared_ptr<marking_map> marking(new marking_map);
2292 imp->get_roster_base(revision_id(curr), *roster, *marking);
2293
2294 for (reconstruction_path::reverse_iterator i = selected_path.rbegin();
2295 i != selected_path.rend(); ++i)
2296 {
2297 id const nxt(*i);
2298 if (global_sanity.debug_p())
2299 L(FL("following delta %s -> %s") % curr % nxt);
2300 roster_delta del;
2301 imp->get_roster_delta(nxt, curr, del);
2302 apply_roster_delta(del, *roster, *marking);
2303 curr = nxt;
2304 }
2305
2306 // Double-check that the thing we got out looks okay. We know that when
2307 // the roster was written to the database, it passed both of these tests,
2308 // and we also know that the data on disk has passed our checks for data
2309 // corruption -- so in theory, we know that what we got out is exactly
2310 // what we put in, and these checks are redundant. (They cannot catch all
2311 // possible errors in any case, e.g., they don't test that the marking is
2312 // correct.) What they can do, though, is serve as a sanity check on the
2313 // delta reconstruction code; if there is a bug where we put something
2314 // into the database and then later get something different back out, then
2315 // this is the only thing that can catch it.
2316 roster->check_sane_against(*marking);
2317 manifest_id expected_mid, actual_mid;
2318 get_revision_manifest(ros_id, expected_mid);
2319 calculate_ident(*roster, actual_mid);
2320 I(expected_mid == actual_mid);
2321
2322 // const'ify the objects, to save them and pass them out
2323 cr.first = roster;
2324 cr.second = marking;
2325 imp->roster_cache.insert_clean(ros_id, cr);
2326}
2327
2328
2329void
2330database_impl::drop(id const & ident,
2331 string const & table)
2332{
2333 string drop = "DELETE FROM " + table + " WHERE id = ?";
2334 execute(query(drop) % blob(ident()));
2335}
2336
2337// ------------------------------------------------------------
2338// -- --
2339// -- public interface follows --
2340// -- --
2341// ------------------------------------------------------------
2342
2343bool
2344database::file_version_exists(file_id const & id)
2345{
2346 return delta_exists(id.inner(), "file_deltas")
2347 || imp->file_or_manifest_base_exists(id.inner(), "files");
2348}
2349
2350bool
2351database::file_size_exists(file_id const & ident)
2352{
2353 return imp->table_has_entry(ident.inner(), "id", "file_sizes");
2354}
2355
2356bool
2357database::roster_version_exists(revision_id const & id)
2358{
2359 return delta_exists(id.inner(), "roster_deltas")
2360 || imp->roster_base_available(id);
2361}
2362
2363bool
2364database::revision_exists(revision_id const & id)
2365{
2366 results res;
2367 query q("SELECT id FROM revisions WHERE id = ?");
2368 imp->fetch(res, one_col, any_rows, q % blob(id.inner()()));
2369 I(res.size() <= 1);
2370 return res.size() == 1;
2371}
2372
2373void
2374database::get_file_ids(set<file_id> & ids)
2375{
2376 ids.clear();
2377 set<id> tmp;
2378 imp->get_ids("files", tmp);
2379 imp->get_ids("file_deltas", tmp);
2380 add_decoration_to_container(tmp, ids);
2381}
2382
2383void
2384database::get_revision_ids(set<revision_id> & ids)
2385{
2386 ids.clear();
2387 set<id> tmp;
2388 imp->get_ids("revisions", tmp);
2389 add_decoration_to_container(tmp, ids);
2390}
2391
2392void
2393database::get_roster_ids(set<revision_id> & ids)
2394{
2395 ids.clear();
2396 set<id> tmp;
2397 imp->get_ids("rosters", tmp);
2398 add_decoration_to_container(tmp, ids);
2399 imp->get_ids("roster_deltas", tmp);
2400 add_decoration_to_container(tmp, ids);
2401}
2402
2403void
2404database::get_file_version(file_id const & id,
2405 file_data & dat)
2406{
2407 data tmp;
2408 imp->get_version(id.inner(), tmp, "files", "file_deltas");
2409 dat = file_data(tmp);
2410}
2411
2412void
2413database::get_file_size(file_id const & ident,
2414 file_size & size)
2415{
2416 results res;
2417 query q("SELECT size FROM file_sizes WHERE id = ?");
2418 imp->fetch(res, one_col, one_row, q % blob(ident.inner()()));
2419 I(!res.empty());
2420 size = lexical_cast<u64>(res[0][0]);
2421}
2422
2423void
2424database::get_file_sizes(roster_t const & roster,
2425 map<file_id, file_size> & sizes)
2426{
2427 sizes.clear();
2428
2429 vector<file_id> all_file_ids;
2430 node_map const & nodes = roster.all_nodes();
2431 for (node_map::const_iterator i = nodes.begin(); i != nodes.end(); ++i)
2432 {
2433 node_id nid = i->first;
2434 if (!is_file_t(i->second))
2435 continue;
2436
2437 file_t file = downcast_to_file_t(i->second);
2438 // filtering out already existing file ids make the whole
2439 // process slower than "accidentially" querying a double
2440 // file id later twice or thrice
2441 all_file_ids.push_back(file->content);
2442 }
2443
2444 // The overall runtime does not improve significantly after ~15, so
2445 // 20 is a good guess. Note that large numbers over 1000 might even
2446 // lead to sqlite errors like "too many SQL max_variables"
2447 size_t max_variables = 20;
2448 for (size_t i = 0; i < all_file_ids.size(); )
2449 {
2450 results res;
2451
2452 size_t variables = all_file_ids.size() - i > max_variables
2453 ? max_variables
2454 : all_file_ids.size() - i;
2455 I(variables > 0);
2456
2457 query q;
2458 string placeholders = "";
2459 for (size_t j=i; j< i + variables; ++j)
2460 {
2461 placeholders += "?,";
2462 q.args.push_back(blob(all_file_ids[j].inner()()));
2463 }
2464
2465 q.sql_cmd = "SELECT id, size FROM file_sizes "
2466 "WHERE id IN(" + placeholders +"null)";
2467
2468 imp->fetch(res, 2, any_rows, q);
2469 I(!res.empty());
2470
2471 for (size_t k=0; k<res.size(); ++k)
2472 {
2473 file_id ident(res[k][0], origin::database);
2474 u64 size = lexical_cast<u64>(res[k][1]);
2475 sizes.insert(make_pair(ident, size));
2476 }
2477
2478 i+= variables;
2479 }
2480}
2481
2482void
2483database::get_manifest_version(manifest_id const & id,
2484 manifest_data & dat)
2485{
2486 data tmp;
2487 imp->get_version(id.inner(), tmp, "manifests", "manifest_deltas");
2488 dat = manifest_data(tmp);
2489}
2490
2491void
2492database::put_file(file_id const & id,
2493 file_data const & dat)
2494{
2495 if (file_version_exists(id))
2496 {
2497 L(FL("file version '%s' already exists in db") % id);
2498 return;
2499 }
2500 imp->schedule_delayed_file(id, dat);
2501 imp->put_file_size(id, dat);
2502}
2503
2504void
2505database::put_file_version(file_id const & old_id,
2506 file_id const & new_id,
2507 file_delta const & del)
2508{
2509 I(!(old_id == new_id));
2510
2511 if (!file_version_exists(old_id))
2512 {
2513 W(F("file preimage '%s' missing in db") % old_id);
2514 W(F("dropping delta '%s' -> '%s'") % old_id % new_id);
2515 return;
2516 }
2517
2518 var_value delta_direction("reverse");
2519 var_key key(var_domain("database"), var_name("delta-direction"));
2520 if (var_exists(key))
2521 {
2522 get_var(key, delta_direction);
2523 }
2524 bool make_reverse_deltas(delta_direction() == "reverse" ||
2525 delta_direction() == "both");
2526 bool make_forward_deltas(delta_direction() == "forward" ||
2527 delta_direction() == "both");
2528 if (!make_reverse_deltas && !make_forward_deltas)
2529 {
2530 W(F("unknown delta direction '%s'; assuming 'reverse'. Valid "
2531 "values are 'reverse', 'forward', 'both'.") % delta_direction);
2532 make_reverse_deltas = true;
2533 }
2534
2535 file_data old_data, new_data;
2536 get_file_version(old_id, old_data);
2537 {
2538 data tmp;
2539 patch(old_data.inner(), del.inner(), tmp);
2540 new_data = file_data(tmp);
2541 }
2542
2543 file_delta reverse_delta;
2544 {
2545 string tmp;
2546 invert_xdelta(old_data.inner()(), del.inner()(), tmp);
2547 reverse_delta = file_delta(tmp, origin::database);
2548 data old_tmp;
2549 patch(new_data.inner(), reverse_delta.inner(), old_tmp);
2550 // We already have the real old data, so compare the
2551 // reconstruction to that directly instead of hashing
2552 // the reconstruction and comparing hashes.
2553 I(old_tmp == old_data.inner());
2554 }
2555
2556 transaction_guard guard(*this);
2557 if (make_reverse_deltas)
2558 {
2559 if (!file_or_manifest_base_exists(new_id, "files"))
2560 {
2561 imp->schedule_delayed_file(new_id, new_data);
2562 imp->put_file_size(new_id, new_data);
2563 }
2564 if (!imp->delta_exists(old_id, new_id, "file_deltas"))
2565 {
2566 put_file_delta(old_id, new_id, reverse_delta);
2567 }
2568 }
2569 if (make_forward_deltas)
2570 {
2571 if (!file_or_manifest_base_exists(new_id, "files"))
2572 {
2573 imp->put_file_size(new_id, new_data);
2574 }
2575 if (!imp->delta_exists(new_id, old_id, "file_deltas"))
2576 {
2577 put_file_delta(new_id, old_id, del);
2578 }
2579 }
2580 else
2581 {
2582 imp->drop(new_id.inner(), "file_deltas");
2583 }
2584 if (file_or_manifest_base_exists(old_id, "files"))
2585 {
2586 // descendent of a head version replaces the head, therefore old head
2587 // must be disposed of
2588 if (delta_exists(old_id.inner(), "file_deltas"))
2589 imp->drop_or_cancel_file(old_id);
2590 }
2591 guard.commit();
2592}
2593
2594void
2595database::get_arbitrary_file_delta(file_id const & src_id,
2596 file_id const & dst_id,
2597 file_delta & del)
2598{
2599 delta dtmp;
2600 // Deltas stored in the database go from base -> id.
2601 results res;
2602 query q1("SELECT delta FROM file_deltas "
2603 "WHERE base = ? AND id = ?");
2604 imp->fetch(res, one_col, any_rows,
2605 q1 % blob(src_id.inner()()) % blob(dst_id.inner()()));
2606
2607 if (!res.empty())
2608 {
2609 // Exact hit: a plain delta from src -> dst.
2610 gzip<delta> del_packed(res[0][0], origin::database);
2611 decode_gzip(del_packed, dtmp);
2612 del = file_delta(dtmp);
2613 return;
2614 }
2615
2616 query q2("SELECT delta FROM file_deltas "
2617 "WHERE base = ? AND id = ?");
2618 imp->fetch(res, one_col, any_rows,
2619 q2 % blob(dst_id.inner()()) % blob(src_id.inner()()));
2620
2621 if (!res.empty())
2622 {
2623 // We have a delta from dst -> src; we need to
2624 // invert this to a delta from src -> dst.
2625 gzip<delta> del_packed(res[0][0], origin::database);
2626 decode_gzip(del_packed, dtmp);
2627 string fwd_delta;
2628 file_data dst;
2629 get_file_version(dst_id, dst);
2630 invert_xdelta(dst.inner()(), dtmp(), fwd_delta);
2631 del = file_delta(fwd_delta, origin::database);
2632 return;
2633 }
2634
2635 // No deltas of use; just load both versions and diff.
2636 file_data fd1, fd2;
2637 get_file_version(src_id, fd1);
2638 get_file_version(dst_id, fd2);
2639 diff(fd1.inner(), fd2.inner(), dtmp);
2640 del = file_delta(dtmp);
2641}
2642
2643
2644void
2645database::get_forward_ancestry(rev_ancestry_map & graph)
2646{
2647 // share some storage
2648 id::symtab id_syms;
2649
2650 results res;
2651 graph.clear();
2652 imp->fetch(res, 2, any_rows,
2653 query("SELECT parent,child FROM revision_ancestry"));
2654 for (size_t i = 0; i < res.size(); ++i)
2655 graph.insert(make_pair(revision_id(res[i][0], origin::database),
2656 revision_id(res[i][1], origin::database)));
2657}
2658
2659void
2660database::get_reverse_ancestry(rev_ancestry_map & graph)
2661{
2662 // share some storage
2663 id::symtab id_syms;
2664
2665 results res;
2666 graph.clear();
2667 imp->fetch(res, 2, any_rows,
2668 query("SELECT child,parent FROM revision_ancestry"));
2669 for (size_t i = 0; i < res.size(); ++i)
2670 graph.insert(make_pair(revision_id(res[i][0], origin::database),
2671 revision_id(res[i][1], origin::database)));
2672}
2673
2674void
2675database::get_revision_parents(revision_id const & id,
2676 set<revision_id> & parents)
2677{
2678 I(!null_id(id));
2679 parent_id_map::iterator i = imp->parent_cache.find(id);
2680 if (i == imp->parent_cache.end())
2681 {
2682 results res;
2683 parents.clear();
2684 imp->fetch(res, one_col, any_rows,
2685 query("SELECT parent FROM revision_ancestry WHERE child = ?")
2686 % blob(id.inner()()));
2687 for (size_t i = 0; i < res.size(); ++i)
2688 parents.insert(revision_id(res[i][0], origin::database));
2689
2690 imp->parent_cache.insert(make_pair(id, parents));
2691 }
2692 else
2693 {
2694 parents = i->second;
2695 }
2696}
2697
2698void
2699database::get_revision_children(revision_id const & id,
2700 set<revision_id> & children)
2701{
2702 results res;
2703 children.clear();
2704 imp->fetch(res, one_col, any_rows,
2705 query("SELECT child FROM revision_ancestry WHERE parent = ?")
2706 % blob(id.inner()()));
2707 for (size_t i = 0; i < res.size(); ++i)
2708 children.insert(revision_id(res[i][0], origin::database));
2709}
2710
2711void
2712database::get_leaves(set<revision_id> & leaves)
2713{
2714 results res;
2715 leaves.clear();
2716 imp->fetch(res, one_col, any_rows,
2717 query("SELECT revisions.id FROM revisions "
2718 "LEFT JOIN revision_ancestry "
2719 "ON revisions.id = revision_ancestry.parent "
2720 "WHERE revision_ancestry.child IS null"));
2721 for (size_t i = 0; i < res.size(); ++i)
2722 leaves.insert(revision_id(res[i][0], origin::database));
2723}
2724
2725
2726void
2727database::get_revision_manifest(revision_id const & rid,
2728 manifest_id & mid)
2729{
2730 revision_t rev;
2731 get_revision(rid, rev);
2732 mid = rev.new_manifest;
2733}
2734
2735void
2736database::get_common_ancestors(std::set<revision_id> const & revs,
2737 std::set<revision_id> & common_ancestors)
2738{
2739 set<revision_id> ancestors, all_common_ancestors;
2740 vector<revision_id> frontier;
2741 for (set<revision_id>::const_iterator i = revs.begin();
2742 i != revs.end(); ++i)
2743 {
2744 I(revision_exists(*i));
2745 ancestors.clear();
2746 ancestors.insert(*i);
2747 frontier.push_back(*i);
2748 while (!frontier.empty())
2749 {
2750 revision_id rid = frontier.back();
2751 frontier.pop_back();
2752 if(!null_id(rid))
2753 {
2754 set<revision_id> parents;
2755 get_revision_parents(rid, parents);
2756 for (set<revision_id>::const_iterator i = parents.begin();
2757 i != parents.end(); ++i)
2758 {
2759 if (ancestors.find(*i) == ancestors.end())
2760 {
2761 frontier.push_back(*i);
2762 ancestors.insert(*i);
2763 }
2764 }
2765 }
2766 }
2767 if (all_common_ancestors.empty())
2768 all_common_ancestors = ancestors;
2769 else
2770 {
2771 set<revision_id> common;
2772 set_intersection(ancestors.begin(), ancestors.end(),
2773 all_common_ancestors.begin(), all_common_ancestors.end(),
2774 inserter(common, common.begin()));
2775 all_common_ancestors = common;
2776 }
2777 }
2778
2779 for (set<revision_id>::const_iterator i = all_common_ancestors.begin();
2780 i != all_common_ancestors.end(); ++i)
2781 {
2782 // null id's here come from the empty parents of root revisions.
2783 // these should not be considered as common ancestors and are skipped.
2784 if (null_id(*i)) continue;
2785 common_ancestors.insert(*i);
2786 }
2787}
2788
2789bool
2790database::is_a_ancestor_of_b(revision_id const & ancestor,
2791 revision_id const & child)
2792{
2793 if (ancestor == child)
2794 return false;
2795
2796 rev_height anc_height;
2797 rev_height child_height;
2798 get_rev_height(ancestor, anc_height);
2799 get_rev_height(child, child_height);
2800
2801 if (anc_height > child_height)
2802 return false;
2803
2804
2805 vector<revision_id> todo;
2806 todo.push_back(ancestor);
2807 set<revision_id> seen;
2808 while (!todo.empty())
2809 {
2810 revision_id anc = todo.back();
2811 todo.pop_back();
2812 set<revision_id> anc_children;
2813 get_revision_children(anc, anc_children);
2814 for (set<revision_id>::const_iterator i = anc_children.begin();
2815 i != anc_children.end(); ++i)
2816 {
2817 if (*i == child)
2818 return true;
2819 else if (seen.find(*i) != seen.end())
2820 continue;
2821 else
2822 {
2823 get_rev_height(*i, anc_height);
2824 if (child_height > anc_height)
2825 {
2826 seen.insert(*i);
2827 todo.push_back(*i);
2828 }
2829 }
2830 }
2831 }
2832 return false;
2833}
2834
2835void
2836database::get_revision(revision_id const & id,
2837 revision_t & rev)
2838{
2839 revision_data d;
2840 get_revision(id, d);
2841 read_revision(d, rev);
2842}
2843
2844void
2845database::get_revision(revision_id const & id,
2846 revision_data & dat)
2847{
2848 I(!null_id(id));
2849 results res;
2850 imp->fetch(res, one_col, one_row,
2851 query("SELECT data FROM revisions WHERE id = ?")
2852 % blob(id.inner()()));
2853 I(res.size() == 1);
2854 gzip<data> gzdata(res[0][0], origin::database);
2855 data rdat;
2856 decode_gzip(gzdata,rdat);
2857
2858 // verify that we got a revision with the right id
2859 {
2860 revision_id tmp;
2861 calculate_ident(revision_data(rdat), tmp);
2862 E(id == tmp, origin::database,
2863 F("revision does not match hash"));
2864 }
2865
2866 dat = revision_data(rdat);
2867}
2868
2869void
2870database::get_rev_height(revision_id const & id,
2871 rev_height & height)
2872{
2873 if (null_id(id))
2874 {
2875 height = rev_height::root_height();
2876 return;
2877 }
2878
2879 height_map::const_iterator i = imp->height_cache.find(id);
2880 if (i == imp->height_cache.end())
2881 {
2882 results res;
2883 imp->fetch(res, one_col, one_row,
2884 query("SELECT height FROM heights WHERE revision = ?")
2885 % blob(id.inner()()));
2886
2887 I(res.size() == 1);
2888
2889 height = rev_height(res[0][0]);
2890 imp->height_cache.insert(make_pair(id, height));
2891 }
2892 else
2893 {
2894 height = i->second;
2895 }
2896
2897 I(height.valid());
2898}
2899
2900void
2901database::put_rev_height(revision_id const & id,
2902 rev_height const & height)
2903{
2904 I(!null_id(id));
2905 I(revision_exists(id));
2906 I(height.valid());
2907
2908 imp->height_cache.erase(id);
2909
2910 imp->execute(query("INSERT INTO heights VALUES(?, ?)")
2911 % blob(id.inner()())
2912 % blob(height()));
2913}
2914
2915bool
2916database::has_rev_height(rev_height const & height)
2917{
2918 results res;
2919 imp->fetch(res, one_col, any_rows,
2920 query("SELECT height FROM heights WHERE height = ?")
2921 % blob(height()));
2922 I((res.size() == 1) || (res.empty()));
2923 return res.size() == 1;
2924}
2925
2926void
2927database::deltify_revision(revision_id const & rid)
2928{
2929 transaction_guard guard(*this);
2930 revision_t rev;
2931 MM(rev);
2932 MM(rid);
2933 get_revision(rid, rev);
2934 // Make sure that all parent revs have their files replaced with deltas
2935 // from this rev's files.
2936 {
2937 for (edge_map::const_iterator i = rev.edges.begin();
2938 i != rev.edges.end(); ++i)
2939 {
2940 for (map<file_path, pair<file_id, file_id> >::const_iterator
2941 j = edge_changes(i).deltas_applied.begin();
2942 j != edge_changes(i).deltas_applied.end(); ++j)
2943 {
2944 file_id old_id(delta_entry_src(j));
2945 file_id new_id(delta_entry_dst(j));
2946 // if not yet deltified
2947 if (file_or_manifest_base_exists(old_id, "files") &&
2948 file_version_exists(new_id))
2949 {
2950 file_data old_data;
2951 file_data new_data;
2952 get_file_version(old_id, old_data);
2953 get_file_version(new_id, new_data);
2954 delta delt;
2955 diff(old_data.inner(), new_data.inner(), delt);
2956 file_delta del(delt);
2957 imp->drop_or_cancel_file(new_id);
2958 imp->drop(new_id.inner(), "file_deltas");
2959 put_file_version(old_id, new_id, del);
2960 }
2961 }
2962 }
2963 }
2964 guard.commit();
2965}
2966
2967
2968bool
2969database::put_revision(revision_id const & new_id,
2970 revision_t const & rev)
2971{
2972 MM(new_id);
2973 MM(rev);
2974
2975 I(!null_id(new_id));
2976
2977 if (revision_exists(new_id))
2978 {
2979 if (global_sanity.debug_p())
2980 L(FL("revision '%s' already exists in db") % new_id);
2981 return false;
2982 }
2983
2984 I(rev.made_for == made_for_database);
2985 rev.check_sane();
2986
2987 // Phase 1: confirm the revision makes sense, and the required files
2988 // actually exist
2989 for (edge_map::const_iterator i = rev.edges.begin();
2990 i != rev.edges.end(); ++i)
2991 {
2992 if (!edge_old_revision(i).inner()().empty()
2993 && !revision_exists(edge_old_revision(i)))
2994 {
2995 W(F("missing prerequisite revision %s")
2996 % edge_old_revision(i));
2997 W(F("dropping revision %s") % new_id);
2998 return false;
2999 }
3000
3001 for (map<file_path, file_id>::const_iterator a
3002 = edge_changes(i).files_added.begin();
3003 a != edge_changes(i).files_added.end(); ++a)
3004 {
3005 if (! file_version_exists(a->second))
3006 {
3007 W(F("missing prerequisite file %s") % a->second);
3008 W(F("dropping revision %s") % new_id);
3009 return false;
3010 }
3011 }
3012
3013 for (map<file_path, pair<file_id, file_id> >::const_iterator d
3014 = edge_changes(i).deltas_applied.begin();
3015 d != edge_changes(i).deltas_applied.end(); ++d)
3016 {
3017 I(!delta_entry_src(d).inner()().empty());
3018 I(!delta_entry_dst(d).inner()().empty());
3019
3020 if (! file_version_exists(delta_entry_src(d)))
3021 {
3022 W(F("missing prerequisite file pre-delta %s")
3023 % delta_entry_src(d));
3024 W(F("dropping revision %s") % new_id);
3025 return false;
3026 }
3027
3028 if (! file_version_exists(delta_entry_dst(d)))
3029 {
3030 W(F("missing prerequisite file post-delta %s")
3031 % delta_entry_dst(d));
3032 W(F("dropping revision %s") % new_id);
3033 return false;
3034 }
3035 }
3036 }
3037
3038 transaction_guard guard(*this);
3039
3040 // Phase 2: Write the revision data (inside a transaction)
3041
3042 revision_data d;
3043 write_revision(rev, d);
3044 gzip<data> d_packed;
3045 encode_gzip(d.inner(), d_packed);
3046 imp->execute(query("INSERT INTO revisions VALUES(?, ?)")
3047 % blob(new_id.inner()())
3048 % blob(d_packed()));
3049
3050 for (edge_map::const_iterator e = rev.edges.begin();
3051 e != rev.edges.end(); ++e)
3052 {
3053 imp->execute(query("INSERT INTO revision_ancestry VALUES(?, ?)")
3054 % blob(edge_old_revision(e).inner()())
3055 % blob(new_id.inner()()));
3056 }
3057 // We don't have to clear out the child's entry in the parent_cache,
3058 // because the child did not exist before this function was called, so
3059 // it can't be in the parent_cache already.
3060
3061 // Phase 3: Construct and write the roster (which also checks the manifest
3062 // id as it goes), but only if the roster does not already exist in the db
3063 // (i.e. because it was left over by a kill_rev_locally)
3064 // FIXME: there is no knowledge yet on speed implications for commands which
3065 // put a lot of revisions in a row (i.e. tailor or cvs_import)!
3066
3067 if (!roster_version_exists(new_id))
3068 {
3069 put_roster_for_revision(new_id, rev);
3070 }
3071 else
3072 {
3073 L(FL("roster for revision '%s' already exists in db") % new_id);
3074 }
3075
3076 // Phase 4: rewrite any files that need deltas added
3077
3078 deltify_revision(new_id);
3079
3080 // Phase 5: determine the revision height
3081
3082 put_height_for_revision(new_id, rev);
3083
3084 // Finally, commit.
3085
3086 guard.commit();
3087 return true;
3088}
3089
3090void
3091database::put_height_for_revision(revision_id const & new_id,
3092 revision_t const & rev)
3093{
3094 I(!null_id(new_id));
3095
3096 rev_height highest_parent;
3097 // we always branch off the highest parent ...
3098 for (edge_map::const_iterator e = rev.edges.begin();
3099 e != rev.edges.end(); ++e)
3100 {
3101 rev_height parent; MM(parent);
3102 get_rev_height(edge_old_revision(e), parent);
3103 if (parent > highest_parent)
3104 {
3105 highest_parent = parent;
3106 }
3107 }
3108
3109 // ... then find the first unused child
3110 u32 childnr(0);
3111 rev_height candidate; MM(candidate);
3112 while(true)
3113 {
3114 candidate = highest_parent.child_height(childnr);
3115 if (!has_rev_height(candidate))
3116 {
3117 break;
3118 }
3119 I(childnr < std::numeric_limits<u32>::max());
3120 ++childnr;
3121 }
3122 put_rev_height(new_id, candidate);
3123}
3124
3125void
3126database::put_file_sizes_for_revision(revision_t const & rev)
3127{
3128 for (edge_map::const_iterator i = rev.edges.begin(); i != rev.edges.end(); ++i)
3129 {
3130 cset const & cs = edge_changes(*i);
3131
3132 for (map<file_path, file_id>::const_iterator i = cs.files_added.begin();
3133 i != cs.files_added.end(); ++i)
3134 {
3135 file_data dat;
3136 get_file_version(i->second, dat);
3137 imp->put_file_size(i->second, dat);
3138 }
3139
3140 for (map<file_path, pair<file_id, file_id> >::const_iterator
3141 i = cs.deltas_applied.begin(); i != cs.deltas_applied.end(); ++i)
3142 {
3143 file_data dat;
3144 get_file_version(i->second.second, dat);
3145 imp->put_file_size(i->second.second, dat);
3146 }
3147 }
3148}
3149
3150void
3151database::put_roster_for_revision(revision_id const & new_id,
3152 revision_t const & rev)
3153{
3154 // Construct, the roster, sanity-check the manifest id, and then write it
3155 // to the db
3156 shared_ptr<roster_t> ros_writeable(new roster_t); MM(*ros_writeable);
3157 shared_ptr<marking_map> mm_writeable(new marking_map); MM(*mm_writeable);
3158 manifest_id roster_manifest_id;
3159 MM(roster_manifest_id);
3160 make_roster_for_revision(*this, rev, new_id, *ros_writeable, *mm_writeable);
3161 calculate_ident(*ros_writeable, roster_manifest_id, false);
3162 E(rev.new_manifest == roster_manifest_id, rev.made_from,
3163 F("revision contains incorrect manifest_id"));
3164 // const'ify the objects, suitable for caching etc.
3165 roster_t_cp ros = ros_writeable;
3166 marking_map_cp mm = mm_writeable;
3167 put_roster(new_id, rev, ros, mm);
3168}
3169
3170bool
3171database::put_revision(revision_id const & new_id,
3172 revision_data const & dat)
3173{
3174 revision_t rev;
3175 read_revision(dat, rev);
3176 return put_revision(new_id, rev);
3177}
3178
3179
3180void
3181database::delete_existing_revs_and_certs()
3182{
3183 imp->execute(query("DELETE FROM revisions"));
3184 imp->execute(query("DELETE FROM revision_ancestry"));
3185 imp->execute(query("DELETE FROM revision_certs"));
3186 imp->execute(query("DELETE FROM branch_leaves"));
3187}
3188
3189void
3190database::delete_existing_manifests()
3191{
3192 imp->execute(query("DELETE FROM manifests"));
3193 imp->execute(query("DELETE FROM manifest_deltas"));
3194}
3195
3196void
3197database::delete_existing_rosters()
3198{
3199 imp->execute(query("DELETE FROM rosters"));
3200 imp->execute(query("DELETE FROM roster_deltas"));
3201 imp->execute(query("DELETE FROM next_roster_node_number"));
3202}
3203
3204void
3205database::delete_existing_heights()
3206{
3207 imp->execute(query("DELETE FROM heights"));
3208}
3209
3210void
3211database::delete_existing_branch_leaves()
3212{
3213 imp->execute(query("DELETE FROM branch_leaves"));
3214}
3215
3216void
3217database::delete_existing_file_sizes()
3218{
3219 imp->execute(query("DELETE FROM file_sizes"));
3220}
3221
3222/// Deletes one revision from the local database.
3223/// @see kill_rev_locally
3224void
3225database::delete_existing_rev_and_certs(revision_id const & rid)
3226{
3227 transaction_guard guard (*this);
3228
3229 // Check that the revision exists and doesn't have any children.
3230 I(revision_exists(rid));
3231 set<revision_id> children;
3232 get_revision_children(rid, children);
3233 I(children.empty());
3234
3235
3236 L(FL("Killing revision %s locally") % rid);
3237
3238 // Kill the certs, ancestry, and revision.
3239 imp->execute(query("DELETE from revision_certs WHERE revision_id = ?")
3240 % blob(rid.inner()()));
3241 {
3242 results res;
3243 imp->fetch(res, one_col, any_rows,
3244 query("SELECT branch FROM branch_leaves where revision_id = ?")
3245 % blob(rid.inner()()));
3246 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3247 {
3248 recalc_branch_leaves(cert_value((*i)[0], origin::database));
3249 }
3250 }
3251 imp->cert_stamper.note_change();
3252
3253 imp->execute(query("DELETE from revision_ancestry WHERE child = ?")
3254 % blob(rid.inner()()));
3255
3256 imp->execute(query("DELETE from heights WHERE revision = ?")
3257 % blob(rid.inner()()));
3258
3259 imp->execute(query("DELETE from revisions WHERE id = ?")
3260 % blob(rid.inner()()));
3261
3262 guard.commit();
3263}
3264
3265void
3266database::compute_branch_leaves(cert_value const & branch_name, set<revision_id> & revs)
3267{
3268 imp->execute(query("DELETE FROM branch_leaves WHERE branch = ?") % blob(branch_name()));
3269 get_revisions_with_cert(cert_name("branch"), branch_name, revs);
3270 erase_ancestors(*this, revs);
3271}
3272
3273void
3274database::recalc_branch_leaves(cert_value const & branch_name)
3275{
3276 imp->execute(query("DELETE FROM branch_leaves WHERE branch = ?") % blob(branch_name()));
3277 set<revision_id> revs;
3278 compute_branch_leaves(branch_name, revs);
3279 for (set<revision_id>::const_iterator i = revs.begin(); i != revs.end(); ++i)
3280 {
3281 imp->execute(query("INSERT INTO branch_leaves (branch, revision_id) "
3282 "VALUES (?, ?)") % blob(branch_name()) % blob((*i).inner()()));
3283 }
3284}
3285
3286void database::delete_certs_locally(revision_id const & rev,
3287 cert_name const & name)
3288{
3289 imp->execute(query("DELETE FROM revision_certs WHERE revision_id = ? AND name = ?")
3290 % blob(rev.inner()()) % text(name()));
3291 imp->cert_stamper.note_change();
3292}
3293void database::delete_certs_locally(revision_id const & rev,
3294 cert_name const & name,
3295 cert_value const & value)
3296{
3297 imp->execute(query("DELETE FROM revision_certs WHERE revision_id = ? AND name = ? AND value = ?")
3298 % blob(rev.inner()()) % text(name()) % blob(value()));
3299 imp->cert_stamper.note_change();
3300}
3301
3302// crypto key management
3303
3304void
3305database::get_key_ids(vector<key_id> & pubkeys)
3306{
3307 pubkeys.clear();
3308 results res;
3309
3310 imp->fetch(res, one_col, any_rows, query("SELECT id FROM public_keys"));
3311
3312 for (size_t i = 0; i < res.size(); ++i)
3313 pubkeys.push_back(key_id(res[i][0], origin::database));
3314}
3315
3316void
3317database_impl::get_keys(string const & table, vector<key_name> & keys)
3318{
3319 keys.clear();
3320 results res;
3321 fetch(res, one_col, any_rows, query("SELECT id FROM " + table));
3322 for (size_t i = 0; i < res.size(); ++i)
3323 keys.push_back(key_name(res[i][0], origin::database));
3324}
3325
3326void
3327database::get_public_keys(vector<key_name> & keys)
3328{
3329 imp->get_keys("public_keys", keys);
3330}
3331
3332bool
3333database::public_key_exists(key_id const & hash)
3334{
3335 MM(hash);
3336 results res;
3337 imp->fetch(res, one_col, any_rows,
3338 query("SELECT id FROM public_keys WHERE id = ?")
3339 % blob(hash.inner()()));
3340 I((res.size() == 1) || (res.empty()));
3341 if (res.size() == 1)
3342 return true;
3343 return false;
3344}
3345
3346bool
3347database::public_key_exists(key_name const & id)
3348{
3349 MM(id);
3350 results res;
3351 imp->fetch(res, one_col, any_rows,
3352 query("SELECT id FROM public_keys WHERE name = ?")
3353 % text(id()));
3354 I((res.size() == 1) || (res.empty()));
3355 if (res.size() == 1)
3356 return true;
3357 return false;
3358}
3359
3360void
3361database::get_pubkey(key_id const & hash,
3362 key_name & id,
3363 rsa_pub_key & pub)
3364{
3365 MM(hash);
3366 results res;
3367 imp->fetch(res, 2, one_row,
3368 query("SELECT name, keydata FROM public_keys WHERE id = ?")
3369 % blob(hash.inner()()));
3370 id = key_name(res[0][0], origin::database);
3371 pub = rsa_pub_key(res[0][1], origin::database);
3372}
3373
3374void
3375database::get_key(key_id const & pub_id,
3376 rsa_pub_key & pub)
3377{
3378 MM(pub_id);
3379 results res;
3380 imp->fetch(res, one_col, one_row,
3381 query("SELECT keydata FROM public_keys WHERE id = ?")
3382 % blob(pub_id.inner()()));
3383 pub = rsa_pub_key(res[0][0], origin::database);
3384}
3385
3386bool
3387database::put_key(key_name const & pub_id,
3388 rsa_pub_key const & pub)
3389{
3390 MM(pub_id);
3391 MM(pub);
3392 key_id thash;
3393 key_hash_code(pub_id, pub, thash);
3394
3395 if (public_key_exists(thash))
3396 {
3397 L(FL("skipping existing public key %s") % pub_id);
3398 return false;
3399 }
3400
3401 L(FL("putting public key %s") % pub_id);
3402
3403 imp->execute(query("INSERT INTO public_keys(id, name, keydata) VALUES(?, ?, ?)")
3404 % blob(thash.inner()())
3405 % text(pub_id())
3406 % blob(pub()));
3407
3408 return true;
3409}
3410
3411void
3412database::delete_public_key(key_id const & pub_id)
3413{
3414 MM(pub_id);
3415 imp->execute(query("DELETE FROM public_keys WHERE id = ?")
3416 % blob(pub_id.inner()()));
3417}
3418
3419void
3420database::encrypt_rsa(key_id const & pub_id,
3421 string const & plaintext,
3422 rsa_oaep_sha_data & ciphertext)
3423{
3424 MM(pub_id);
3425 rsa_pub_key pub;
3426 get_key(pub_id, pub);
3427
3428 SecureVector<Botan::byte> pub_block
3429 (reinterpret_cast<Botan::byte const *>(pub().data()), pub().size());
3430
3431 shared_ptr<X509_PublicKey> x509_key(Botan::X509::load_key(pub_block));
3432 shared_ptr<RSA_PublicKey> pub_key
3433 = shared_dynamic_cast<RSA_PublicKey>(x509_key);
3434 if (!pub_key)
3435 throw recoverable_failure(origin::system,
3436 "Failed to get RSA encrypting key");
3437
3438 shared_ptr<PK_Encryptor>
3439 encryptor(get_pk_encryptor(*pub_key, "EME1(SHA-1)"));
3440
3441 SecureVector<Botan::byte> ct;
3442
3443#if BOTAN_VERSION_CODE >= BOTAN_VERSION_CODE_FOR(1,7,7)
3444 ct = encryptor->encrypt(
3445 reinterpret_cast<Botan::byte const *>(plaintext.data()),
3446 plaintext.size(), lazy_rng::get());
3447#else
3448 ct = encryptor->encrypt(
3449 reinterpret_cast<Botan::byte const *>(plaintext.data()),
3450 plaintext.size());
3451#endif
3452 ciphertext = rsa_oaep_sha_data(string(reinterpret_cast<char const *>(ct.begin()),
3453 ct.size()),
3454 origin::database);
3455}
3456
3457cert_status
3458database::check_signature(key_id const & id,
3459 string const & alleged_text,
3460 rsa_sha1_signature const & signature)
3461{
3462 MM(id);
3463 MM(alleged_text);
3464 shared_ptr<PK_Verifier> verifier;
3465
3466 verifier_cache::const_iterator i = imp->verifiers.find(id);
3467 if (i != imp->verifiers.end())
3468 verifier = i->second.first;
3469
3470 else
3471 {
3472 rsa_pub_key pub;
3473
3474 if (!public_key_exists(id))
3475 return cert_unknown;
3476
3477 get_key(id, pub);
3478 SecureVector<Botan::byte> pub_block
3479 (reinterpret_cast<Botan::byte const *>(pub().data()), pub().size());
3480
3481 L(FL("building verifier for %d-byte pub key") % pub_block.size());
3482 shared_ptr<X509_PublicKey> x509_key(Botan::X509::load_key(pub_block));
3483 shared_ptr<RSA_PublicKey> pub_key
3484 = boost::shared_dynamic_cast<RSA_PublicKey>(x509_key);
3485
3486 E(pub_key, id.inner().made_from,
3487 F("failed to get RSA verifying key for %s") % id);
3488
3489 verifier.reset(get_pk_verifier(*pub_key, "EMSA3(SHA-1)"));
3490
3491 /* XXX This is ugly. We need to keep the key around
3492 * as long as the verifier is around, but the shared_ptr will go
3493 * away after we leave this scope. Hence we store a pair of
3494 * <verifier,key> so they both exist. */
3495 imp->verifiers.insert(make_pair(id, make_pair(verifier, pub_key)));
3496 }
3497
3498 // check the text+sig against the key
3499 L(FL("checking %d-byte signature") % signature().size());
3500
3501 if (verifier->verify_message(
3502 reinterpret_cast<Botan::byte const*>(alleged_text.data()),
3503 alleged_text.size(),
3504 reinterpret_cast<Botan::byte const*>(signature().data()),
3505 signature().size()))
3506 return cert_ok;
3507 else
3508 return cert_bad;
3509}
3510
3511cert_status
3512database::check_cert(cert const & t)
3513{
3514 string signed_text;
3515 t.signable_text(signed_text);
3516 return check_signature(t.key, signed_text, t.sig);
3517}
3518
3519// cert management
3520
3521bool
3522database_impl::cert_exists(cert const & t,
3523 string const & table)
3524{
3525 results res;
3526 query q = query("SELECT revision_id FROM " + table + " WHERE revision_id = ? "
3527 "AND name = ? "
3528 "AND value = ? "
3529 "AND keypair_id = ? "
3530 "AND signature = ?")
3531 % blob(t.ident.inner()())
3532 % text(t.name())
3533 % blob(t.value())
3534 % blob(t.key.inner()())
3535 % blob(t.sig());
3536
3537 fetch(res, 1, any_rows, q);
3538
3539 I(res.empty() || res.size() == 1);
3540 return res.size() == 1;
3541}
3542
3543void
3544database_impl::put_cert(cert const & t,
3545 string const & table)
3546{
3547 results res;
3548 fetch(res, 1, one_row,
3549 query("SELECT name FROM public_keys WHERE id = ?")
3550 % blob(t.key.inner()()));
3551 key_name keyname(res[0][0], origin::database);
3552
3553 id thash;
3554 t.hash_code(keyname, thash);
3555 rsa_sha1_signature sig;
3556
3557 string insert = "INSERT INTO " + table + " VALUES(?, ?, ?, ?, ?, ?)";
3558
3559 execute(query(insert)
3560 % blob(thash())
3561 % blob(t.ident.inner()())
3562 % text(t.name())
3563 % blob(t.value())
3564 % blob(t.key.inner()())
3565 % blob(t.sig()));
3566}
3567
3568void
3569database_impl::results_to_certs(results const & res,
3570 vector<cert> & certs)
3571{
3572 certs.clear();
3573 for (size_t i = 0; i < res.size(); ++i)
3574 {
3575 cert t;
3576 t = cert(revision_id(res[i][0], origin::database),
3577 cert_name(res[i][1], origin::database),
3578 cert_value(res[i][2], origin::database),
3579 key_id(res[i][3], origin::database),
3580 rsa_sha1_signature(res[i][4], origin::database));
3581 certs.push_back(t);
3582 }
3583}
3584
3585void
3586database_impl::results_to_certs(results const & res,
3587 vector<pair<id, cert> > & certs)
3588{
3589 certs.clear();
3590 for (size_t i = 0; i < res.size(); ++i)
3591 {
3592 cert t;
3593 t = cert(revision_id(res[i][0], origin::database),
3594 cert_name(res[i][1], origin::database),
3595 cert_value(res[i][2], origin::database),
3596 key_id(res[i][3], origin::database),
3597 rsa_sha1_signature(res[i][4], origin::database));
3598 certs.push_back(make_pair(id(res[i][5], origin::database),
3599 t));
3600 }
3601}
3602
3603void
3604database_impl::oldstyle_results_to_certs(results const & res,
3605 vector<cert> & certs)
3606{
3607 certs.clear();
3608 for (size_t i = 0; i < res.size(); ++i)
3609 {
3610 revision_id rev_id(res[i][0], origin::database);
3611 cert_name name(res[i][1], origin::database);
3612 cert_value value(res[i][2], origin::database);
3613
3614 key_name k_name(res[i][3], origin::database);
3615 key_id k_id;
3616 {
3617 results key_res;
3618 query lookup_key("SELECT id FROM public_keys WHERE name = ?");
3619 fetch(key_res, 1, any_rows, lookup_key % text(k_name()));
3620 if (key_res.size() == 0)
3621 break; // no key, cert is bogus
3622 else if (key_res.size() == 1)
3623 k_id = key_id(key_res[0][0], origin::database);
3624 else
3625 E(false, origin::database,
3626 F("your database contains multiple keys named '%s'") % k_name);
3627 }
3628
3629 rsa_sha1_signature sig(res[i][4], origin::database);
3630 certs.push_back(cert(rev_id, name, value, k_id, sig));
3631 }
3632}
3633
3634void
3635database_impl::install_functions()
3636{
3637#ifdef SUPPORT_SQLITE_BEFORE_3003014
3638 if (sqlite3_libversion_number() < 3003013)
3639 I(sqlite3_create_function(sql(), "hex", -1,
3640 SQLITE_UTF8, NULL,
3641 &sqlite3_hex_fn,
3642 NULL, NULL) == 0);
3643#endif
3644
3645 // register any functions we're going to use
3646 I(sqlite3_create_function(sql(), "gunzip", -1,
3647 SQLITE_UTF8, NULL,
3648 &sqlite3_gunzip_fn,
3649 NULL, NULL) == 0);
3650}
3651
3652void
3653database_impl::get_certs(vector<cert> & certs,
3654 string const & table)
3655{
3656 results res;
3657 query q("SELECT revision_id, name, value, keypair_id, signature FROM " + table);
3658 fetch(res, 5, any_rows, q);
3659 results_to_certs(res, certs);
3660}
3661
3662
3663void
3664database_impl::get_oldstyle_certs(id const & ident,
3665 vector<cert> & certs,
3666 string const & table)
3667{
3668 MM(ident);
3669 results res;
3670 query q("SELECT id, name, value, keypair, signature FROM " + table +
3671 " WHERE id = ?");
3672
3673 fetch(res, 5, any_rows, q % blob(ident()));
3674 oldstyle_results_to_certs(res, certs);
3675}
3676
3677void
3678database_impl::get_certs(id const & ident,
3679 vector<cert> & certs,
3680 string const & table)
3681{
3682 MM(ident);
3683 results res;
3684 query q("SELECT revision_id, name, value, keypair_id, signature FROM " + table +
3685 " WHERE revision_id = ?");
3686
3687 fetch(res, 5, any_rows, q % blob(ident()));
3688 results_to_certs(res, certs);
3689}
3690
3691void
3692database_impl::get_certs(cert_name const & name,
3693 vector<cert> & certs,
3694 string const & table)
3695{
3696 MM(name);
3697 results res;
3698 query q("SELECT revision_id, name, value, keypair_id, signature FROM " + table +
3699 " WHERE name = ?");
3700 fetch(res, 5, any_rows, q % text(name()));
3701 results_to_certs(res, certs);
3702}
3703
3704void
3705database_impl::get_oldstyle_certs(cert_name const & name,
3706 vector<cert> & certs,
3707 string const & table)
3708{
3709 results res;
3710 query q("SELECT id, name, value, keypair, signature FROM " + table +
3711 " WHERE name = ?");
3712 fetch(res, 5, any_rows, q % text(name()));
3713 oldstyle_results_to_certs(res, certs);
3714}
3715
3716void
3717database_impl::get_certs(id const & ident,
3718 cert_name const & name,
3719 vector<cert> & certs,
3720 string const & table)
3721{
3722 results res;
3723 query q("SELECT revision_id, name, value, keypair_id, signature FROM " + table +
3724 " WHERE revision_id = ? AND name = ?");
3725
3726 fetch(res, 5, any_rows,
3727 q % blob(ident())
3728 % text(name()));
3729 results_to_certs(res, certs);
3730}
3731
3732void
3733database_impl::get_certs(cert_name const & name,
3734 cert_value const & val,
3735 vector<pair<id, cert> > & certs,
3736 string const & table)
3737{
3738 results res;
3739 query q("SELECT revision_id, name, value, keypair_id, signature, hash FROM " + table +
3740 " WHERE name = ? AND value = ?");
3741
3742 fetch(res, 6, any_rows,
3743 q % text(name())
3744 % blob(val()));
3745 results_to_certs(res, certs);
3746}
3747
3748
3749void
3750database_impl::get_certs(id const & ident,
3751 cert_name const & name,
3752 cert_value const & value,
3753 vector<cert> & certs,
3754 string const & table)
3755{
3756 results res;
3757 query q("SELECT revision_id, name, value, keypair_id, signature FROM " + table +
3758 " WHERE revision_id = ? AND name = ? AND value = ?");
3759
3760 fetch(res, 5, any_rows,
3761 q % blob(ident())
3762 % text(name())
3763 % blob(value()));
3764 results_to_certs(res, certs);
3765}
3766
3767
3768
3769bool
3770database::revision_cert_exists(cert const & cert)
3771{
3772 return imp->cert_exists(cert, "revision_certs");
3773}
3774
3775bool
3776database::put_revision_cert(cert const & cert)
3777{
3778 if (revision_cert_exists(cert))
3779 {
3780 L(FL("revision cert on '%s' already exists in db")
3781 % cert.ident);
3782 return false;
3783 }
3784
3785 if (!revision_exists(revision_id(cert.ident)))
3786 {
3787 W(F("cert revision %s does not exist in db")
3788 % cert.ident);
3789 W(F("dropping cert"));
3790 return false;
3791 }
3792
3793 if (cert.name() == "branch")
3794 {
3795 string branch_name = cert.value();
3796 if (branch_name.find_first_of("?,;*%%+{}[]!^") != string::npos ||
3797 branch_name.find_first_of('-') == 0)
3798 {
3799 W(F("the branch name\n"
3800 " '%s'\n"
3801 "contains meta characters (one or more of '?,;*%%+{}[]!^') or\n"
3802 "starts with a dash, which might cause malfunctions when used\n"
3803 "in a netsync branch pattern.\n\n"
3804 "If you want to undo this operation, please use the\n"
3805 "'%s local kill_certs' command to delete the particular branch\n"
3806 "cert and re-add a valid one.")
3807 % cert.value() % prog_name);
3808 }
3809 }
3810
3811 imp->put_cert(cert, "revision_certs");
3812
3813 if (cert.name() == "branch")
3814 {
3815 record_as_branch_leaf(cert.value, cert.ident);
3816 }
3817
3818 imp->cert_stamper.note_change();
3819 return true;
3820}
3821
3822void
3823database::record_as_branch_leaf(cert_value const & branch, revision_id const & rev)
3824{
3825 set<revision_id> parents;
3826 get_revision_parents(rev, parents);
3827 set<revision_id> current_leaves;
3828 get_branch_leaves(branch, current_leaves);
3829
3830 set<revision_id>::const_iterator self = current_leaves.find(rev);
3831 if (self != current_leaves.end())
3832 return; // already recorded (must be adding a second branch cert)
3833
3834 bool all_parents_were_leaves = true;
3835 bool some_ancestor_was_leaf = false;
3836 for (set<revision_id>::const_iterator p = parents.begin();
3837 p != parents.end(); ++p)
3838 {
3839 set<revision_id>::iterator l = current_leaves.find(*p);
3840 if (l == current_leaves.end())
3841 all_parents_were_leaves = false;
3842 else
3843 {
3844 some_ancestor_was_leaf = true;
3845 imp->execute(query("DELETE FROM branch_leaves "
3846 "WHERE branch = ? AND revision_id = ?")
3847 % blob(branch()) % blob(l->inner()()));
3848 current_leaves.erase(l);
3849 }
3850 }
3851
3852 // This check is needed for this case:
3853 //
3854 // r1 (branch1)
3855 // |
3856 // r2 (branch2)
3857 // |
3858 // r3 (branch1)
3859
3860 if (!all_parents_were_leaves)
3861 {
3862 for (set<revision_id>::const_iterator r = current_leaves.begin();
3863 r != current_leaves.end(); ++r)
3864 {
3865 if (is_a_ancestor_of_b(*r, rev))
3866 {
3867 some_ancestor_was_leaf = true;
3868 imp->execute(query("DELETE FROM branch_leaves "
3869 "WHERE branch = ? AND revision_id = ?")
3870 % blob(branch()) % blob(r->inner()()));
3871 }
3872 }
3873 }
3874
3875 // are we really a leaf (ie, not an ancestor of an existing leaf)?
3876 //
3877 // see tests/branch_leaves_sync_bug for a scenario that requires this.
3878 if (!some_ancestor_was_leaf)
3879 {
3880 bool really_a_leaf = true;
3881 for (set<revision_id>::const_iterator r = current_leaves.begin();
3882 r != current_leaves.end(); ++r)
3883 {
3884 if (is_a_ancestor_of_b(rev, *r))
3885 {
3886 really_a_leaf = false;
3887 break;
3888 }
3889 }
3890 if (!really_a_leaf)
3891 return;
3892 }
3893
3894 imp->execute(query("INSERT INTO branch_leaves(branch, revision_id) "
3895 "VALUES (?, ?)")
3896 % blob(branch()) % blob(rev.inner()()));
3897}
3898
3899outdated_indicator
3900database::get_revision_cert_nobranch_index(vector< pair<revision_id,
3901 pair<revision_id, key_id> > > & idx)
3902{
3903 // share some storage
3904 id::symtab id_syms;
3905
3906 results res;
3907 imp->fetch(res, 3, any_rows,
3908 query("SELECT hash, revision_id, keypair_id "
3909 "FROM revision_certs WHERE name != 'branch'"));
3910
3911 idx.clear();
3912 idx.reserve(res.size());
3913 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3914 {
3915 idx.push_back(make_pair(revision_id((*i)[0], origin::database),
3916 make_pair(revision_id((*i)[1], origin::database),
3917 key_id((*i)[2], origin::database))));
3918 }
3919 return imp->cert_stamper.get_indicator();
3920}
3921
3922outdated_indicator
3923database::get_revision_certs(vector<cert> & certs)
3924{
3925 imp->get_certs(certs, "revision_certs");
3926 return imp->cert_stamper.get_indicator();
3927}
3928
3929outdated_indicator
3930database::get_revision_certs(cert_name const & name,
3931 vector<cert> & certs)
3932{
3933 imp->get_certs(name, certs, "revision_certs");
3934 return imp->cert_stamper.get_indicator();
3935}
3936
3937outdated_indicator
3938database::get_revision_certs(revision_id const & id,
3939 cert_name const & name,
3940 vector<cert> & certs)
3941{
3942 imp->get_certs(id.inner(), name, certs, "revision_certs");
3943 return imp->cert_stamper.get_indicator();
3944}
3945
3946outdated_indicator
3947database::get_revision_certs(revision_id const & id,
3948 cert_name const & name,
3949 cert_value const & val,
3950 vector<cert> & certs)
3951{
3952 imp->get_certs(id.inner(), name, val, certs, "revision_certs");
3953 return imp->cert_stamper.get_indicator();
3954}
3955
3956outdated_indicator
3957database::get_revisions_with_cert(cert_name const & name,
3958 cert_value const & val,
3959 set<revision_id> & revisions)
3960{
3961 revisions.clear();
3962 results res;
3963 query q("SELECT revision_id FROM revision_certs WHERE name = ? AND value = ?");
3964 imp->fetch(res, one_col, any_rows, q % text(name()) % blob(val()));
3965 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3966 revisions.insert(revision_id((*i)[0], origin::database));
3967 return imp->cert_stamper.get_indicator();
3968}
3969
3970outdated_indicator
3971database::get_branch_leaves(cert_value const & value,
3972 set<revision_id> & revisions)
3973{
3974 revisions.clear();
3975 results res;
3976 query q("SELECT revision_id FROM branch_leaves WHERE branch = ?");
3977 imp->fetch(res, one_col, any_rows, q % blob(value()));
3978 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
3979 revisions.insert(revision_id((*i)[0], origin::database));
3980 return imp->cert_stamper.get_indicator();
3981}
3982
3983outdated_indicator
3984database::get_revision_certs(cert_name const & name,
3985 cert_value const & val,
3986 vector<pair<id, cert> > & certs)
3987{
3988 imp->get_certs(name, val, certs, "revision_certs");
3989 return imp->cert_stamper.get_indicator();
3990}
3991
3992outdated_indicator
3993database::get_revision_certs(revision_id const & id,
3994 vector<cert> & certs)
3995{
3996 imp->get_certs(id.inner(), certs, "revision_certs");
3997 return imp->cert_stamper.get_indicator();
3998}
3999
4000outdated_indicator
4001database::get_revision_certs(revision_id const & ident,
4002 vector<id> & ids)
4003{
4004 results res;
4005 imp->fetch(res, one_col, any_rows,
4006 query("SELECT hash "
4007 "FROM revision_certs "
4008 "WHERE revision_id = ?")
4009 % blob(ident.inner()()));
4010 ids.clear();
4011 for (size_t i = 0; i < res.size(); ++i)
4012 ids.push_back(id(res[i][0], origin::database));
4013 return imp->cert_stamper.get_indicator();
4014}
4015
4016void
4017database::get_revision_cert(id const & hash,
4018 cert & c)
4019{
4020 results res;
4021 vector<cert> certs;
4022 imp->fetch(res, 5, one_row,
4023 query("SELECT revision_id, name, value, keypair_id, signature "
4024 "FROM revision_certs "
4025 "WHERE hash = ?")
4026 % blob(hash()));
4027 imp->results_to_certs(res, certs);
4028 I(certs.size() == 1);
4029 c = certs[0];
4030}
4031
4032bool
4033database::revision_cert_exists(revision_id const & hash)
4034{
4035 results res;
4036 vector<cert> certs;
4037 imp->fetch(res, one_col, any_rows,
4038 query("SELECT revision_id "
4039 "FROM revision_certs "
4040 "WHERE hash = ?")
4041 % blob(hash.inner()()));
4042 I(res.empty() || res.size() == 1);
4043 return (res.size() == 1);
4044}
4045
4046// FIXME: the bogus-cert family of functions is ridiculous
4047// and needs to be replaced, or at least factored.
4048namespace {
4049 struct trust_value
4050 {
4051 set<key_id> good_sigs;
4052 set<key_id> bad_sigs;
4053 set<key_id> unknown_sigs;
4054 };
4055
4056 // returns *one* of each trusted cert key/value
4057 // if two keys signed the same thing, we get two certs as input and
4058 // just pick one (assuming neither is invalid) to use in the output
4059 void
4060 erase_bogus_certs_internal(vector<cert> & certs,
4061 database & db,
4062 database::cert_trust_checker const & checker)
4063 {
4064 // sorry, this is a crazy data structure
4065 typedef tuple<id, cert_name, cert_value> trust_key;
4066 typedef map< trust_key, trust_value > trust_map;
4067 trust_map trust;
4068
4069 for (vector<cert>::iterator i = certs.begin(); i != certs.end(); ++i)
4070 {
4071 trust_key key = trust_key(i->ident.inner(),
4072 i->name,
4073 i->value);
4074 trust_value & value = trust[key];
4075 switch (db.check_cert(*i))
4076 {
4077 case cert_ok:
4078 value.good_sigs.insert(i->key);
4079 break;
4080 case cert_bad:
4081 value.bad_sigs.insert(i->key);
4082 break;
4083 case cert_unknown:
4084 value.unknown_sigs.insert(i->key);
4085 break;
4086 }
4087 }
4088
4089 certs.clear();
4090
4091 for (trust_map::const_iterator i = trust.begin();
4092 i != trust.end(); ++i)
4093 {
4094 cert out(typecast_vocab<revision_id>(get<0>(i->first)),
4095 get<1>(i->first), get<2>(i->first), key_id());
4096 if (!i->second.good_sigs.empty() &&
4097 checker(i->second.good_sigs,
4098 get<0>(i->first),
4099 get<1>(i->first),
4100 get<2>(i->first)))
4101 {
4102 L(FL("trust function liked %d signers of %s cert on revision %s")
4103 % i->second.good_sigs.size()
4104 % get<1>(i->first)
4105 % get<0>(i->first));
4106 out.key = *i->second.good_sigs.begin();
4107 certs.push_back(out);
4108 }
4109 else
4110 {
4111 string txt;
4112 out.signable_text(txt);
4113 for (set<key_id>::const_iterator b = i->second.bad_sigs.begin();
4114 b != i->second.bad_sigs.end(); ++b)
4115 {
4116 W(F("ignoring bad signature by '%s' on '%s'") % *b % txt);
4117 }
4118 for (set<key_id>::const_iterator u = i->second.unknown_sigs.begin();
4119 u != i->second.unknown_sigs.end(); ++u)
4120 {
4121 W(F("ignoring unknown signature by '%s' on '%s'") % *u % txt);
4122 }
4123 W(F("trust function disliked %d signers of '%s' cert on revision %s")
4124 % i->second.good_sigs.size()
4125 % get<1>(i->first)
4126 % get<0>(i->first));
4127 }
4128 }
4129 }
4130 // the lua hook wants key_identity_info, but all that's been
4131 // pulled from the certs is key_id. So this is needed to translate.
4132 // use pointers for project and lua so bind() doesn't make copies
4133 bool check_revision_cert_trust(project_t const * const project,
4134 lua_hooks * const lua,
4135 set<key_id> const & signers,
4136 id const & hash,
4137 cert_name const & name,
4138 cert_value const & value)
4139 {
4140 set<key_identity_info> signer_identities;
4141 for (set<key_id>::const_iterator i = signers.begin();
4142 i != signers.end(); ++i)
4143 {
4144 key_identity_info identity;
4145 identity.id = *i;
4146 project->complete_key_identity_from_id(*lua, identity);
4147 signer_identities.insert(identity);
4148 }
4149
4150 return lua->hook_get_revision_cert_trust(signer_identities,
4151 hash, name, value);
4152 }
4153 // and the lua hook for manifest trust checking wants a key_name
4154 bool check_manifest_cert_trust(database * const db,
4155 lua_hooks * const lua,
4156 set<key_id> const & signers,
4157 id const & hash,
4158 cert_name const & name,
4159 cert_value const & value)
4160 {
4161 set<key_name> signer_names;
4162 for (set<key_id>::const_iterator i = signers.begin();
4163 i != signers.end(); ++i)
4164 {
4165 key_name name;
4166 rsa_pub_key pub;
4167 db->get_pubkey(*i, name, pub);
4168 signer_names.insert(name);
4169 }
4170
4171 return lua->hook_get_manifest_cert_trust(signer_names,
4172 hash, name, value);
4173 }
4174} // anonymous namespace
4175
4176void
4177database::erase_bogus_certs(project_t const & project, vector<cert> & certs)
4178{
4179 erase_bogus_certs_internal(certs, *this,
4180 boost::bind(&check_revision_cert_trust,
4181 &project, &this->lua, _1, _2, _3, _4));
4182}
4183void
4184database::erase_bogus_certs(vector<cert> & certs,
4185 database::cert_trust_checker const & checker)
4186{
4187 erase_bogus_certs_internal(certs, *this, checker);
4188}
4189
4190// These are only used by migration from old manifest-style ancestry, so we
4191// don't much worry that they are not perfectly typesafe. Also, we know
4192// that the callers want bogus certs erased.
4193
4194void
4195database::get_manifest_certs(manifest_id const & id, std::vector<cert> & certs)
4196{
4197 imp->get_oldstyle_certs(id.inner(), certs, "manifest_certs");
4198 erase_bogus_certs_internal(certs, *this,
4199 boost::bind(&check_manifest_cert_trust,
4200 this, &this->lua, _1, _2, _3, _4));
4201}
4202
4203void
4204database::get_manifest_certs(cert_name const & name, std::vector<cert> & certs)
4205{
4206 imp->get_oldstyle_certs(name, certs, "manifest_certs");
4207 erase_bogus_certs_internal(certs, *this,
4208 boost::bind(&check_manifest_cert_trust,
4209 this, &this->lua, _1, _2, _3, _4));
4210}
4211
4212// completions
4213void
4214database_impl::add_prefix_matching_constraint(string const & colname,
4215 string const & prefix,
4216 query & q)
4217{
4218 L(FL("add_prefix_matching_constraint for '%s'") % prefix);
4219
4220 if (prefix.empty())
4221 q.sql_cmd += "1"; // always true
4222 else if (prefix.size() > constants::idlen)
4223 q.sql_cmd += "0"; // always false
4224 else
4225 {
4226 for (string::const_iterator i = prefix.begin(); i != prefix.end(); ++i)
4227 {
4228 E(is_xdigit(*i), origin::user,
4229 F("bad character '%c' in id name '%s'") % *i % prefix);
4230 }
4231
4232 string lower_hex = prefix;
4233 if (lower_hex.size() < constants::idlen)
4234 lower_hex.append(constants::idlen - lower_hex.size(), '0');
4235 string lower_bound = decode_hexenc(lower_hex, origin::internal);
4236
4237 string upper_hex = prefix;
4238 if (upper_hex.size() < constants::idlen)
4239 upper_hex.append(constants::idlen - upper_hex.size(), 'f');
4240 string upper_bound = decode_hexenc(upper_hex, origin::internal);
4241
4242 if (global_sanity.debug_p())
4243 L(FL("prefix_matcher: lower bound ('%s') and upper bound ('%s')")
4244 % encode_hexenc(lower_bound, origin::internal)
4245 % encode_hexenc(upper_bound, origin::internal));
4246
4247 q.sql_cmd += colname + " BETWEEN ? AND ?";
4248 q.args.push_back(blob(lower_bound));
4249 q.args.push_back(blob(upper_bound));
4250 }
4251}
4252
4253void
4254database::complete(string const & partial,
4255 set<revision_id> & completions)
4256{
4257 results res;
4258 completions.clear();
4259 query q("SELECT id FROM revisions WHERE ");
4260
4261 imp->add_prefix_matching_constraint("id", partial, q);
4262 imp->fetch(res, 1, any_rows, q);
4263
4264 for (size_t i = 0; i < res.size(); ++i)
4265 completions.insert(revision_id(res[i][0], origin::database));
4266}
4267
4268
4269void
4270database::complete(string const & partial,
4271 set<file_id> & completions)
4272{
4273 results res;
4274 completions.clear();
4275
4276 query q("SELECT id FROM files WHERE ");
4277 imp->add_prefix_matching_constraint("id", partial, q);
4278 imp->fetch(res, 1, any_rows, q);
4279
4280 for (size_t i = 0; i < res.size(); ++i)
4281 completions.insert(file_id(res[i][0], origin::database));
4282
4283 res.clear();
4284
4285 q = query("SELECT id FROM file_deltas WHERE ");
4286 imp->add_prefix_matching_constraint("id", partial, q);
4287 imp->fetch(res, 1, any_rows, q);
4288
4289 for (size_t i = 0; i < res.size(); ++i)
4290 completions.insert(file_id(res[i][0], origin::database));
4291}
4292
4293void
4294database::complete(string const & partial,
4295 set< pair<key_id, utf8 > > & completions)
4296{
4297 results res;
4298 completions.clear();
4299 query q("SELECT id, name FROM public_keys WHERE ");
4300
4301 imp->add_prefix_matching_constraint("id", partial, q);
4302 imp->fetch(res, 2, any_rows, q);
4303
4304 for (size_t i = 0; i < res.size(); ++i)
4305 completions.insert(make_pair(key_id(res[i][0], origin::database),
4306 utf8(res[i][1], origin::database)));
4307}
4308
4309// revision selectors
4310
4311void
4312database::select_parent(string const & partial,
4313 set<revision_id> & completions)
4314{
4315 results res;
4316 completions.clear();
4317
4318 query q("SELECT DISTINCT parent FROM revision_ancestry WHERE ");
4319 imp->add_prefix_matching_constraint("child", partial, q);
4320 imp->fetch(res, 1, any_rows, q);
4321
4322 for (size_t i = 0; i < res.size(); ++i)
4323 completions.insert(revision_id(res[i][0], origin::database));
4324}
4325
4326void
4327database::select_cert(string const & certname,
4328 set<revision_id> & completions)
4329{
4330 results res;
4331 completions.clear();
4332
4333 imp->fetch(res, 1, any_rows,
4334 query("SELECT DISTINCT revision_id FROM revision_certs WHERE name = ?")
4335 % text(certname));
4336
4337 for (size_t i = 0; i < res.size(); ++i)
4338 completions.insert(revision_id(res[i][0], origin::database));
4339}
4340
4341void
4342database::select_cert(string const & certname, string const & certvalue,
4343 set<revision_id> & completions)
4344{
4345 results res;
4346 completions.clear();
4347
4348 imp->fetch(res, 1, any_rows,
4349 query("SELECT DISTINCT revision_id FROM revision_certs"
4350 " WHERE name = ? AND CAST(value AS TEXT) GLOB ?")
4351 % text(certname) % text(certvalue));
4352
4353 for (size_t i = 0; i < res.size(); ++i)
4354 completions.insert(revision_id(res[i][0], origin::database));
4355}
4356
4357void
4358database::select_author_tag_or_branch(string const & partial,
4359 set<revision_id> & completions)
4360{
4361 results res;
4362 completions.clear();
4363
4364 string pattern = partial + "*";
4365
4366 imp->fetch(res, 1, any_rows,
4367 query("SELECT DISTINCT revision_id FROM revision_certs"
4368 " WHERE (name=? OR name=? OR name=?)"
4369 " AND CAST(value AS TEXT) GLOB ?")
4370 % text(author_cert_name()) % text(tag_cert_name())
4371 % text(branch_cert_name()) % text(pattern));
4372
4373 for (size_t i = 0; i < res.size(); ++i)
4374 completions.insert(revision_id(res[i][0], origin::database));
4375}
4376
4377void
4378database::select_date(string const & date, string const & comparison,
4379 set<revision_id> & completions)
4380{
4381 results res;
4382 completions.clear();
4383
4384 query q;
4385 q.sql_cmd = ("SELECT DISTINCT revision_id FROM revision_certs "
4386 "WHERE name = ? AND CAST(value AS TEXT) ");
4387 q.sql_cmd += comparison;
4388 q.sql_cmd += " ?";
4389
4390 imp->fetch(res, 1, any_rows,
4391 q % text(date_cert_name()) % text(date));
4392 for (size_t i = 0; i < res.size(); ++i)
4393 completions.insert(revision_id(res[i][0], origin::database));
4394}
4395
4396void
4397database::select_key(key_id const & id, set<revision_id> & completions)
4398{
4399 results res;
4400 completions.clear();
4401
4402 imp->fetch(res, 1, any_rows,
4403 query("SELECT DISTINCT revision_id FROM revision_certs"
4404 " WHERE keypair_id = ?")
4405 % blob(id.inner()()));
4406
4407 for (size_t i = 0; i < res.size(); ++i)
4408 completions.insert(revision_id(res[i][0], origin::database));
4409}
4410
4411// epochs
4412
4413void
4414database::get_epochs(map<branch_name, epoch_data> & epochs)
4415{
4416 epochs.clear();
4417 results res;
4418 imp->fetch(res, 2, any_rows, query("SELECT branch, epoch FROM branch_epochs"));
4419 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
4420 {
4421 branch_name decoded(idx(*i, 0), origin::database);
4422 I(epochs.find(decoded) == epochs.end());
4423 epochs.insert(make_pair(decoded,
4424 epoch_data(idx(*i, 1),
4425 origin::database)));
4426 }
4427}
4428
4429void
4430database::get_epoch(epoch_id const & eid,
4431 branch_name & branch, epoch_data & epo)
4432{
4433 I(epoch_exists(eid));
4434 results res;
4435 imp->fetch(res, 2, any_rows,
4436 query("SELECT branch, epoch FROM branch_epochs"
4437 " WHERE hash = ?")
4438 % blob(eid.inner()()));
4439 I(res.size() == 1);
4440 branch = branch_name(idx(idx(res, 0), 0), origin::database);
4441 epo = epoch_data(idx(idx(res, 0), 1), origin::database);
4442}
4443
4444bool
4445database::epoch_exists(epoch_id const & eid)
4446{
4447 results res;
4448 imp->fetch(res, one_col, any_rows,
4449 query("SELECT hash FROM branch_epochs WHERE hash = ?")
4450 % blob(eid.inner()()));
4451 I(res.size() == 1 || res.empty());
4452 return res.size() == 1;
4453}
4454
4455void
4456database::set_epoch(branch_name const & branch, epoch_data const & epo)
4457{
4458 epoch_id eid;
4459 epoch_hash_code(branch, epo, eid);
4460 I(epo.inner()().size() == constants::epochlen_bytes);
4461 imp->execute(query("INSERT OR REPLACE INTO branch_epochs VALUES(?, ?, ?)")
4462 % blob(eid.inner()())
4463 % blob(branch())
4464 % blob(epo.inner()()));
4465}
4466
4467void
4468database::clear_epoch(branch_name const & branch)
4469{
4470 imp->execute(query("DELETE FROM branch_epochs WHERE branch = ?")
4471 % blob(branch()));
4472}
4473
4474bool
4475database::check_integrity()
4476{
4477 results res;
4478 imp->fetch(res, one_col, any_rows, query("PRAGMA integrity_check"));
4479 I(res.size() == 1);
4480 I(res[0].size() == 1);
4481
4482 return res[0][0] == "ok";
4483}
4484
4485// vars
4486
4487void
4488database::get_vars(map<var_key, var_value> & vars)
4489{
4490 vars.clear();
4491 results res;
4492 imp->fetch(res, 3, any_rows, query("SELECT domain, name, value FROM db_vars"));
4493 for (results::const_iterator i = res.begin(); i != res.end(); ++i)
4494 {
4495 var_domain domain(idx(*i, 0), origin::database);
4496 var_name name(idx(*i, 1), origin::database);
4497 var_value value(idx(*i, 2), origin::database);
4498 I(vars.find(make_pair(domain, name)) == vars.end());
4499 vars.insert(make_pair(make_pair(domain, name), value));
4500 }
4501}
4502
4503void
4504database::get_var(var_key const & key, var_value & value)
4505{
4506 results res;
4507 imp->fetch(res, one_col, any_rows,
4508 query("SELECT value FROM db_vars "
4509 "WHERE domain = ? AND name = ?")
4510 % text(key.first())
4511 % blob(key.second()));
4512 I(res.size() == 1);
4513 var_value dbvalue(res[0][0], origin::database);
4514 value = dbvalue;
4515}
4516
4517bool
4518database::var_exists(var_key const & key)
4519{
4520 results res;
4521 imp->fetch(res, one_col, any_rows,
4522 query("SELECT 1 "
4523 "WHERE EXISTS("
4524 " SELECT 1 FROM db_vars "
4525 " WHERE domain = ? AND name = ?)")
4526 % text(key.first())
4527 % blob(key.second()));
4528 return ! res.empty();
4529}
4530
4531void
4532database::set_var(var_key const & key, var_value const & value)
4533{
4534 imp->execute(query("INSERT OR REPLACE INTO db_vars VALUES(?, ?, ?)")
4535 % text(key.first())
4536 % blob(key.second())
4537 % blob(value()));
4538}
4539
4540void
4541database::clear_var(var_key const & key)
4542{
4543 imp->execute(query("DELETE FROM db_vars WHERE domain = ? AND name = ?")
4544 % text(key.first())
4545 % blob(key.second()));
4546}
4547
4548#define KNOWN_WORKSPACES_KEY \
4549 var_key(make_pair( \
4550 var_domain("database", origin::internal), \
4551 var_name("known-workspaces", origin::internal) \
4552 ))
4553
4554void
4555database::register_workspace(system_path const & workspace)
4556{
4557 var_value val;
4558 if (var_exists(KNOWN_WORKSPACES_KEY))
4559 get_var(KNOWN_WORKSPACES_KEY, val);
4560
4561 vector<string> workspaces;
4562 split_into_lines(val(), workspaces);
4563
4564 vector<string>::iterator pos =
4565 find(workspaces.begin(),
4566 workspaces.end(),
4567 workspace.as_internal());
4568 if (pos == workspaces.end())
4569 workspaces.push_back(workspace.as_internal());
4570
4571 string ws;
4572 join_lines(workspaces, ws);
4573
4574 set_var(KNOWN_WORKSPACES_KEY, var_value(ws, origin::internal));
4575}
4576
4577void
4578database::unregister_workspace(system_path const & workspace)
4579{
4580 if (var_exists(KNOWN_WORKSPACES_KEY))
4581 {
4582 var_value val;
4583 get_var(KNOWN_WORKSPACES_KEY, val);
4584
4585 vector<string> workspaces;
4586 split_into_lines(val(), workspaces);
4587
4588 vector<string>::iterator pos =
4589 find(workspaces.begin(),
4590 workspaces.end(),
4591 workspace.as_internal());
4592 if (pos != workspaces.end())
4593 workspaces.erase(pos);
4594
4595 string ws;
4596 join_lines(workspaces, ws);
4597
4598 set_var(KNOWN_WORKSPACES_KEY, var_value(ws, origin::internal));
4599 }
4600}
4601
4602void
4603database::get_registered_workspaces(vector<system_path> & workspaces)
4604{
4605 if (var_exists(KNOWN_WORKSPACES_KEY))
4606 {
4607 var_value val;
4608 get_var(KNOWN_WORKSPACES_KEY, val);
4609
4610 vector<string> paths;
4611 split_into_lines(val(), paths);
4612
4613 for (vector<string>::const_iterator i = paths.begin();
4614 i != paths.end(); ++i)
4615 {
4616 system_path workspace_path(*i, origin::database);
4617 workspaces.push_back(workspace_path);
4618 }
4619 }
4620}
4621
4622void
4623database::set_registered_workspaces(vector<system_path> const & workspaces)
4624{
4625 vector<string> paths;
4626 for (vector<system_path>::const_iterator i = workspaces.begin();
4627 i != workspaces.end(); ++i)
4628 {
4629 paths.push_back((*i).as_internal());
4630 }
4631
4632 string ws;
4633 join_lines(paths, ws);
4634 set_var(KNOWN_WORKSPACES_KEY, var_value(ws, origin::internal));
4635}
4636
4637#undef KNOWN_WORKSPACES_KEY
4638
4639// branches
4640
4641outdated_indicator
4642database::get_branches(vector<string> & names)
4643{
4644 results res;
4645 query q("SELECT DISTINCT branch FROM branch_leaves");
4646 string cert_name = "branch";
4647 imp->fetch(res, one_col, any_rows, q);
4648 for (size_t i = 0; i < res.size(); ++i)
4649 {
4650 names.push_back(res[i][0]);
4651 }
4652 return imp->cert_stamper.get_indicator();
4653}
4654
4655outdated_indicator
4656database::get_branches(globish const & glob,
4657 vector<string> & names)
4658{
4659 results res;
4660 query q("SELECT DISTINCT value FROM revision_certs WHERE name = ?");
4661 string cert_name = "branch";
4662 imp->fetch(res, one_col, any_rows, q % text(cert_name));
4663 for (size_t i = 0; i < res.size(); ++i)
4664 {
4665 if (glob.matches(res[i][0]))
4666 names.push_back(res[i][0]);
4667 }
4668 return imp->cert_stamper.get_indicator();
4669}
4670
4671void
4672database::get_roster(revision_id const & rev_id,
4673 roster_t & roster)
4674{
4675 marking_map mm;
4676 get_roster(rev_id, roster, mm);
4677}
4678
4679void
4680database::get_roster(revision_id const & rev_id,
4681 roster_t & roster,
4682 marking_map & marking)
4683{
4684 if (rev_id.inner()().empty())
4685 {
4686 roster = roster_t();
4687 marking = marking_map();
4688 return;
4689 }
4690
4691 cached_roster cr;
4692 get_roster(rev_id, cr);
4693 roster = *cr.first;
4694 marking = *cr.second;
4695}
4696
4697void
4698database::get_roster(revision_id const & rev_id, cached_roster & cr)
4699{
4700 get_roster_version(rev_id, cr);
4701 I(cr.first);
4702 I(cr.second);
4703}
4704
4705void
4706database::put_roster(revision_id const & rev_id,
4707 revision_t const & rev,
4708 roster_t_cp const & roster,
4709 marking_map_cp const & marking)
4710{
4711 I(roster);
4712 I(marking);
4713 MM(rev_id);
4714
4715 transaction_guard guard(*this);
4716
4717 // Our task is to add this roster, and deltify all the incoming edges (if
4718 // they aren't already).
4719
4720 imp->roster_cache.insert_dirty(rev_id, make_pair(roster, marking));
4721
4722 // Now do what deltify would do if we bothered
4723 size_t num_edges = rev.edges.size();
4724 for (edge_map::const_iterator i = rev.edges.begin();
4725 i != rev.edges.end(); ++i)
4726 {
4727 revision_id old_rev = edge_old_revision(*i);
4728 if (null_id(old_rev))
4729 continue;
4730 if (imp->roster_base_stored(old_rev))
4731 {
4732 cached_roster cr;
4733 get_roster_version(old_rev, cr);
4734 roster_delta reverse_delta;
4735 cset const & changes = edge_changes(i);
4736 delta_rosters(*roster, *marking,
4737 *(cr.first), *(cr.second),
4738 reverse_delta,
4739 num_edges > 1 ? 0 : &changes);
4740 if (imp->roster_cache.exists(old_rev))
4741 imp->roster_cache.mark_clean(old_rev);
4742 imp->drop(old_rev.inner(), "rosters");
4743 imp->put_roster_delta(old_rev, rev_id, reverse_delta);
4744 }
4745 }
4746 guard.commit();
4747}
4748
4749// for get_uncommon_ancestors
4750struct rev_height_graph : rev_graph
4751{
4752 rev_height_graph(database & db) : db(db) {}
4753 virtual void get_parents(revision_id const & rev, set<revision_id> & parents) const
4754 {
4755 db.get_revision_parents(rev, parents);
4756 }
4757 virtual void get_children(revision_id const & rev, set<revision_id> & parents) const
4758 {
4759 // not required
4760 I(false);
4761 }
4762 virtual void get_height(revision_id const & rev, rev_height & h) const
4763 {
4764 db.get_rev_height(rev, h);
4765 }
4766
4767 database & db;
4768};
4769
4770void
4771database::get_uncommon_ancestors(revision_id const & a,
4772 revision_id const & b,
4773 set<revision_id> & a_uncommon_ancs,
4774 set<revision_id> & b_uncommon_ancs)
4775{
4776
4777 rev_height_graph graph(*this);
4778 ::get_uncommon_ancestors(a, b, graph, a_uncommon_ancs, b_uncommon_ancs);
4779}
4780
4781node_id
4782database::next_node_id()
4783{
4784 transaction_guard guard(*this);
4785 results res;
4786
4787 // We implement this as a fixed db var.
4788 imp->fetch(res, one_col, any_rows,
4789 query("SELECT node FROM next_roster_node_number"));
4790
4791 u64 n = 1;
4792 if (res.empty())
4793 {
4794 imp->execute(query("INSERT INTO next_roster_node_number VALUES(1)"));
4795 }
4796 else
4797 {
4798 I(res.size() == 1);
4799 n = lexical_cast<u64>(res[0][0]);
4800 ++n;
4801 imp->execute(query("UPDATE next_roster_node_number SET node = ?")
4802 % text(lexical_cast<string>(n)));
4803 }
4804 guard.commit();
4805 return static_cast<node_id>(n);
4806}
4807
4808void
4809database_impl::check_filename()
4810{
4811 E(!filename.empty(), origin::user, F("no database specified"));
4812}
4813
4814
4815void
4816database_impl::check_db_exists()
4817{
4818 switch (get_path_status(filename))
4819 {
4820 case path::file:
4821 return;
4822
4823 case path::nonexistent:
4824 E(false, origin::user, F("database '%s' does not exist") % filename);
4825
4826 case path::directory:
4827 if (directory_is_workspace(filename))
4828 {
4829 options opts;
4830 workspace::get_options(filename, opts);
4831 E(opts.dbname.as_internal().empty(), origin::user,
4832 F("'%s' is a workspace, not a database\n"
4833 "(did you mean '%s'?)") % filename % opts.dbname);
4834 }
4835 E(false, origin::user,
4836 F("'%s' is a directory, not a database") % filename);
4837 }
4838}
4839
4840void
4841database_impl::check_db_nonexistent()
4842{
4843 require_path_is_nonexistent(filename,
4844 F("database '%s' already exists")
4845 % filename);
4846
4847 system_path journal(filename.as_internal() + "-journal", origin::internal);
4848 require_path_is_nonexistent(journal,
4849 F("existing (possibly stale) journal file '%s' "
4850 "has same stem as new database '%s'.\n"
4851 "Cancelling database creation")
4852 % journal % filename);
4853
4854}
4855
4856void
4857database_impl::open()
4858{
4859 I(!__sql);
4860
4861 string to_open;
4862 if (type == memory_db)
4863 to_open = memory_db_identifier;
4864 else
4865 {
4866 system_path base_dir = filename.dirname();
4867 if (!directory_exists(base_dir))
4868 mkdir_p(base_dir);
4869 to_open = filename.as_external();
4870 }
4871
4872 if (sqlite3_open(to_open.c_str(), &__sql) == SQLITE_NOMEM)
4873 throw std::bad_alloc();
4874
4875 I(__sql);
4876 assert_sqlite3_ok(__sql);
4877}
4878
4879void
4880database_impl::close()
4881{
4882 I(__sql);
4883
4884 sqlite3_close(__sql);
4885 __sql = 0;
4886
4887 I(!__sql);
4888}
4889
4890// transaction guards
4891
4892conditional_transaction_guard::~conditional_transaction_guard()
4893{
4894 if (!acquired)
4895 return;
4896 if (committed)
4897 db.imp->commit_transaction();
4898 else
4899 db.imp->rollback_transaction();
4900}
4901
4902void
4903conditional_transaction_guard::acquire()
4904{
4905 I(!acquired);
4906 acquired = true;
4907 db.imp->begin_transaction(exclusive);
4908}
4909
4910void
4911conditional_transaction_guard::do_checkpoint()
4912{
4913 I(acquired);
4914 db.imp->commit_transaction();
4915 db.imp->begin_transaction(exclusive);
4916 checkpointed_calls = 0;
4917 checkpointed_bytes = 0;
4918}
4919
4920void
4921conditional_transaction_guard::maybe_checkpoint(size_t nbytes)
4922{
4923 I(acquired);
4924 checkpointed_calls += 1;
4925 checkpointed_bytes += nbytes;
4926 if (checkpointed_calls >= checkpoint_batch_size
4927 || checkpointed_bytes >= checkpoint_batch_bytes)
4928 do_checkpoint();
4929}
4930
4931void
4932conditional_transaction_guard::commit()
4933{
4934 I(acquired);
4935 committed = true;
4936}
4937
4938void
4939database_path_helper::get_database_path(options const & opts,
4940 system_path & path,
4941 database::dboptions dbopts)
4942{
4943 if (!opts.dbname_given ||
4944 (opts.dbname.as_internal().empty() &&
4945 opts.dbname_alias.empty() &&
4946 opts.dbname_type != memory_db))
4947 {
4948 if (dbopts == database::maybe_unspecified)
4949 {
4950 L(FL("no database option given or options empty"));
4951 return;
4952 }
4953 E(false, origin::user, F("no database specified"));
4954 }
4955
4956 if (opts.dbname_type == unmanaged_db)
4957 {
4958 path = opts.dbname;
4959 return;
4960 }
4961
4962 if (opts.dbname_type == memory_db)
4963 {
4964 return;
4965 }
4966
4967 I(opts.dbname_type == managed_db);
4968
4969 path_component basename;
4970 validate_and_clean_alias(opts.dbname_alias, basename);
4971
4972 vector<system_path> candidates;
4973 vector<system_path> search_paths;
4974
4975 E(lua.hook_get_default_database_locations(search_paths) && search_paths.size() > 0,
4976 origin::user, F("no default database location configured"));
4977
4978 for (vector<system_path>::const_iterator i = search_paths.begin();
4979 i != search_paths.end(); ++i)
4980 {
4981 if (file_exists((*i) / basename))
4982 {
4983 candidates.push_back((*i) / basename);
4984 continue;
4985 }
4986 }
4987
4988 MM(candidates);
4989
4990 // if we did not found the database anywhere, use the first
4991 // available default path to possible save it there
4992 if (candidates.size() == 0)
4993 {
4994 path = (*search_paths.begin()) / basename;
4995 L(FL("no path expansions found for '%s', using '%s'")
4996 % opts.dbname_alias % path);
4997 return;
4998 }
4999
5000 if (candidates.size() == 1)
5001 {
5002 path = (*candidates.begin());
5003 L(FL("one path expansion found for '%s': '%s'")
5004 % opts.dbname_alias % path);
5005 return;
5006 }
5007
5008 if (candidates.size() > 1)
5009 {
5010 string err =
5011 (F("the database alias '%s' has multiple ambiguous expansions:")
5012 % opts.dbname_alias).str();
5013
5014 for (vector<system_path>::const_iterator i = candidates.begin();
5015 i != candidates.end(); ++i)
5016 err += ("\n " + (*i).as_internal());
5017
5018 E(false, origin::user, i18n_format(err));
5019 }
5020}
5021
5022void
5023database_path_helper::maybe_set_default_alias(options & opts)
5024{
5025 if (opts.dbname_given && (
5026 !opts.dbname.as_internal().empty() ||
5027 !opts.dbname_alias.empty()))
5028 {
5029 return;
5030 }
5031
5032 string alias;
5033 E(lua.hook_get_default_database_alias(alias) && !alias.empty(),
5034 origin::user, F("could not query default database alias"));
5035
5036 P(F("using default database '%s'") % alias);
5037 opts.dbname_given = true;
5038 opts.dbname_alias = alias;
5039 opts.dbname_type = managed_db;
5040}
5041
5042void
5043database_path_helper::validate_and_clean_alias(string const & alias, path_component & pc)
5044{
5045 E(alias.find(':') == 0, origin::system,
5046 F("invalid database alias '%s': does not start with a colon") % alias);
5047
5048 string pure_alias = alias.substr(1);
5049 E(pure_alias.size() > 0, origin::system,
5050 F("invalid database alias '%s': must not be empty") % alias);
5051
5052 globish matcher;
5053 E(lua.hook_get_default_database_glob(matcher),
5054 origin::user, F("could not query default database glob"));
5055
5056 if (!matcher.matches(pure_alias))
5057 pure_alias += ".mtn";
5058
5059 try
5060 {
5061 pc = path_component(pure_alias, origin::system);
5062 }
5063 catch (...)
5064 {
5065 E(false, origin::system,
5066 F("invalid database alias '%s': does contain invalid characters") % alias);
5067 }
5068}
5069
5070// Local Variables:
5071// mode: C++
5072// fill-column: 76
5073// c-file-style: "gnu"
5074// indent-tabs-mode: nil
5075// End:
5076// vim: et:sw=2:sts=2:ts=2:cino=>2s,{s,\:s,+s,t0,g0,^-2,e-2,n-2,p2s,(0,=s:

Archive Download this file