mogadm

Revision 1208, 45.8 kB (checked in by dormando, 5 months ago)
update mailing list url.
Property svn:eol-style set to `native` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`

Line
1	#!/usr/bin/perl
2	# vim:ts=4 sw=4 ft=perl et:
3
4	use strict;
5	use warnings;
6	use Getopt::Long;
7	use LWP::Simple; # FIXME: use of this makes 'mog check' hang too long when multiple things down
8	use Socket;
9
10	my @topcmds = qw(check stats host device domain class slave fsck settings);
11	my $usage = {
12	check => {
13	des => "Check the state of the MogileFS world.",
14	},
15	stats => {
16	des => "Show MogileFS system statistics.",
17	},
18	settings => {
19	des => "Change/list server settings.",
20	subcmd => {
21	list => {
22	des => "List all server settings",
23	},
24	set => {
25	args => "<key> <value>",
26	des => "Set server setting 'key' to 'value'.",
27	},
28	},
29	},
30	host => {
31	des => "Add/modify hosts.",
32	subcmd => {
33	list => {
34	des => "List all hosts.",
35	},
36	add => {
37	des => "Add a host to MogileFS.",
38	args => "<hostname> [opts]",
39	opts => {
40	"<hostname>" => "Hostname of machine",
41	"--status=s" => "One of {alive,down}. Default 'down'.",
42	"--ip=s" => "IP address of machine.",
43	"--port=i" => "HTTP port of mogstored",
44	"--getport=i" => "Alternate HTTP port serving readonly traffic",
45	"--altip=s" => "Alternate IP that is machine is reachable from",
46	"--altmask=s" => "Netmask which, when matches client, uses alt IP",
47	},
48	},
49	modify => {
50	des => "Modify a host's properties.",
51	args => "<hostname> [opts]",
52	opts => {
53	"<hostname>" => "Host name.",
54	"--status=s" => "One of {alive,down}.",
55	"--ip=s" => "IP address of machine.",
56	"--port=i" => "HTTP port of mogstored",
57	"--getport=i" => "Alternate HTTP port serving readonly traffic",
58	"--altip=s" => "Alternate IP that is machine is reachable from",
59	"--altmask=s" => "Netmask which, when matches client, uses alt IP",
60	},
61	},
62	mark => {
63	des => "Change the status of a host. (equivalent to 'modify --status')",
64	args => "<hostname> <status>",
65	opts => {
66	"<hostname>" => "Host name to bring up or down.",
67	"<status>" => "One of {alive,down}.",
68	}
69	},
70	delete => {
71	des => "Delete a host.",
72	args => "<hostname>",
73	opts => {
74	"<hostname>" => "Host name to delete.",
75	},
76	},
77	},
78	},
79	device => {
80	des => "Add/modify devices.",
81	subcmd => {
82	list => {
83	des => "List all devices, for each host.",
84	args => "[opts]",
85	opts => {
86	"--all" => "Include dead devices in list.",
87	},
88	},
89	summary => {
90	des => "List the summary of devices, for each host.",
91	args => "[opts]",
92	opts => {
93	"--status=s" => "Devices of status A. Defaults to 'alive,readonly'",
94	},
95	},
96	add => {
97	des => "Add a device to a host.",
98	args => "<hostname> <devid> [opts]",
99	opts => {
100	"<hostname>" => "Hostname to add a device",
101	"<devid>" => "Numeric devid. Never reuse these.",
102	"--status=s" => "One of 'alive' or 'down'. Defaults to 'alive'.",
103	},
104	},
105	mark => {
106	des => "Mark a device as {alive,dead,down,drain,readonly}",
107	args => "<hostname> <devid> <status>",
108	opts => {
109	"<hostname>" => "Hostname of device",
110	"<devid>" => "Numeric devid to modify.",
111	"<status>" => "One of {alive,dead,down,drain,readonly}",
112	},
113	},
114	modify => {
115	des => "Modify a device's properties.",
116	args => "<hostname> <devid> [opts]",
117	opts => {
118	"<hostname>" => "Hostname of device",
119	"<devid>" => "Numeric devid to modify.",
120	"--status=s" => "One of {alive,dead,down,drain,readonly}",
121	"--weight=i" => "Positive numeric weight for device",
122	},
123	},
124	},
125	},
126	domain => {
127	des => "Add/modify domains (namespaces)",
128	subcmd => {
129	list => {
130	des => "List all hosts.",
131	},
132	add => {
133	des => "Add a domain (namespace)",
134	args => "<domain>",
135	opts => {
136	"<domain>" => "Domain (namespace) to add.",
137	},
138	},
139	delete => {
140	des => "Delete a domain.",
141	args => "<domain>",
142	opts => {
143	"<domain>" => "Domain (namespace) to add.",
144	},
145	},
146	},
147	},
148	class => {
149	des => "Add/modify file classes.",
150	subcmd => {
151	list => {
152	des => "List all classes, for each domain.",
153	},
154	add => {
155	des => "Add a file class to a domain.",
156	args => "<domain> <class> [opts]",
157	opts => {
158	"<domain>" => "Domain to add class to.",
159	"<class>" => "Name of class to add.",
160	"--mindevcount=i" => "Minimum number of replicas.",
161	},
162	},
163	modify => {
164	des => "Modify properties of a file class.",
165	args => "<domain> <class> [opts]",
166	opts => {
167	"<domain>" => "Domain to add class to.",
168	"<class>" => "Name of class to add.",
169	"--mindevcount=i" => "Minimum number of replicas.",
170	},
171	},
172	delete => {
173	des => "Delete a file class from a domain.",
174	args => "<domain> <class>",
175	opts => {
176	"<domain>" => "Domain of class to delete.",
177	"<class>" => "Class to delete.",
178	},
179
180	},
181	},
182	},
183	slave => {
184	des => 'Manipulate slave database information in a running mogilefsd.',
185	subcmd => {
186	list => {
187	des => 'List current store slave nodes.',
188	},
189	add => {
190	des => 'Add a slave node for store usage',
191	args => '<slave_key> [opts]',
192	opts => {
193	'--dsn=s' => "DBI DSN specifying what database to connect to.",
194	'--username=s' => "DBI username for connecting.",
195	'--password=s' => "DBI password for connecting.",
196	},
197	},
198	modify => {
199	des => 'Modify a slave node for store usage',
200	args => '<slave_key> [opts]',
201	opts => {
202	'--dsn=s' => "DBI DSN specifying what database to connect to.",
203	'--username=s' => "DBI username for connecting.",
204	'--password=s' => "DBI password for connecting.",
205	},
206	},
207	delete => {
208	des => 'Delete a slave node for store usage',
209	args => '<slave_key>',
210	},
211	},
212	},
213	fsck => {
214	des => "Control a background filesystem check operation.",
215	subcmd => {
216	start => {
217	des => 'Start (or resume) background fsck',
218	},
219	stop => {
220	des => 'Stop (pause) background fsck',
221	},
222	status => {
223	des => 'Show fsck status',
224	},
225	reset => {
226	des => 'Reset fsck position back to the beginning',
227	args => '[opts]',
228	opts => {
229	'--policy-only' => "Check repl policy (assumed locations); don't stat storage nodes",
230	'--startpos=i' => "FID to start at.",
231	}
232	},
233	clearlog => {
234	des => 'Clear the fsck log',
235	},
236	printlog => {
237	des => 'Display the fsck log',
238	},
239	taillog => {
240	des => 'Tail the fsck log',
241	},
242
243	},
244	},
245	};
246
247	# load up our config files
248	my %opts;
249
250	Getopt::Long::Configure("require_order", "pass_through");
251	GetOptions(
252	"trackers=s" => \$opts{trackers},
253	"config=s" => \$opts{config},
254	"lib=s" => \$opts{lib},
255	"help" => \$opts{help},
256	"verbose" => \$opts{verbose},
257	) or abortWithUsage();
258	Getopt::Long::Configure("require_order", "no_pass_through");
259
260	my @configs = ($opts{config}, "$ENV{HOME}/.mogilefs.conf", "/etc/mogilefs/mogilefs.conf");
261	foreach my $fn (reverse @configs) {
262	next unless $fn && -e $fn;
263	open FILE, "<$fn"
264	or die "unable to open $fn: $!\n";
265	while (<FILE>) {
266	s/\#.*//;
267	next unless m!^\s(\w+)\s=\s(.+?)\s$!;
268	$opts{$1} = $2 unless ( defined $opts{$1} );
269	}
270	close FILE;
271	}
272
273	# bail for help
274	abortWithUsage() if $opts{help};
275
276	# make sure we have at least a topcmd
277	my $topcmd = shift(@ARGV);
278	abortWithUsage() unless $topcmd && $usage->{$topcmd};
279
280	# break up the trackers and ensure we got some
281	if ($opts{trackers}) {
282	$opts{trackers} = [ split(/\s,\s/, $opts{trackers}) ];
283	}
284	fail_text('no_trackers')
285	unless ($opts{trackers} && @{$opts{trackers}}) \|\| detect_local_tracker();
286
287	# okay, load up the libraries that we need
288	if ($opts{lib}) {
289	eval "use lib '$opts{lib}';";
290	}
291	eval "use MogileFS::Admin; use MogileFS::Client; 1;" or fail_text('cant_find_module');
292
293	# dispatch if it's special
294	if ($topcmd eq 'check') {
295	die "Unknown options/arguments to 'check' command.\n" if @ARGV;
296	cmd_check();
297	} elsif ($topcmd eq 'stats') {
298	die "Unknown options/arguments to 'stats' command.\n" if @ARGV;
299	cmd_stats();
300	}
301
302	# get the verb
303	my $verb = shift(@ARGV) or
304	abort_with_topcmd_help($topcmd);
305	my $cmdinfo = $usage->{$topcmd}{subcmd}{$verb};
306	abort_with_topcmd_help($topcmd) unless $cmdinfo;
307
308	my $badargs = sub {
309	my $msg = shift;
310	abort_with_topcmd_help($topcmd, $verb, $msg);
311	};
312
313	# get the non-option (non --foo) arguments:
314	my %cmdargs;
315	if (my $args = $cmdinfo->{args}) {
316	my @args = split(/ /, $args);
317	foreach my $arg (@args) {
318	# positional (but named) parameter
319	if ($arg =~ /^<(.+)>$/) {
320	my $argname = $1;
321	my $val = shift @ARGV;
322	# map e.g. "dev5" to 5
323	if ($argname eq "devid" && $val && $val =~ /^dev(\d+)$/) {
324	$val = $1;
325	}
326	$badargs->("Missing argument '$argname'") unless defined $val;
327	$badargs->("Unexpected option. Expected argument '$argname'") if $val =~ /^-/;
328	$cmdargs{$argname} = $val;
329	} elsif ($arg eq "[opts]") {
330	# handled later.
331	} else {
332	die "INTERNAL ERROR.";
333	}
334	}
335	$badargs->("Unexpected extra argument.") if @ARGV && $ARGV[0] !~ /^-/;
336	} else {
337	$badargs->("Unexpected arguments when expecting none.") if @ARGV;
338	}
339
340	# parse the options
341	if (my $opts = $cmdinfo->{opts}) {
342	my %getopts;
343	foreach (keys %$opts) {
344	my $k = $_;
345	next if $k =~ /^</; # don't care about these.
346	die "BOGUS KEY: '$k'" unless $k =~ /^--([\w\-]+)(=.+)?$/;
347	my ($oname, $type) = ($1, $2 \|\| "");
348	$getopts{"$oname$type"} = \$cmdargs{$1};
349	}
350	GetOptions(%getopts)
351	or abort_with_topcmd_help($topcmd, $verb);
352	}
353
354	# see what we should do
355	my $cmdsub = do {
356	no strict 'refs';
357	*{"cmd_${topcmd}_${verb}"} or abortWithUsage();
358	};
359
360	# now call our lovely lovely sub
361	$cmdsub->(\%cmdargs);
362	exit 0;
363
364	sub detect_local_tracker {
365	require IO::Socket::INET;
366	my $loctrack = "127.0.0.1:7001";
367	my $sock = IO::Socket::INET->new(PeerAddr => $loctrack, Timeout => 1);
368	return 0 unless $sock;
369	$opts{trackers} = [$loctrack];
370	return 1;
371	}
372
373	###########################################################################
374	## command routines
375	###########################################################################
376
377	sub cmd_check {
378	# step 1: we want to check each tracker for responsiveness
379	my $now = time();
380	my ($hosts, $devices);
381	$\| = 1;
382	print "Checking trackers...\n";
383	foreach my $t (@{$opts{trackers}}) {
384	print " $t ... ";
385	my $mogadm = mogadm($t);
386	if ($mogadm) {
387	my $lhosts = hosts($mogadm);
388	my $ldevs = devices($mogadm);
389	if ($lhosts && $ldevs) {
390	print "OK\n";
391	$hosts = $lhosts;
392	$devices = $ldevs;
393	} else {
394	print "REQUEST FAILURE (is the tracker up?)\n";
395	}
396	} else {
397	print "INITIAL FAILURE\n";
398	}
399	}
400
401	# we should have hosts if we get here
402	fail_text('no_hosts') unless $hosts;
403	print "\n";
404
405	# step 2: now hit each of the hosts for responsiveness
406	print "Checking hosts...\n";
407	my @urls;
408	foreach my $hostid (sort { $a <=> $b } keys %$hosts) {
409	printf " [%2d] %s ... ", $hostid, $hosts->{$hostid}->{hostname};
410	if ($hosts->{$hostid}->{status} eq 'alive') {
411	my $url = 'http://' . $hosts->{$hostid}->{hostip} . ':' . $hosts->{$hostid}->{http_port} . '/';
412	my $file = get($url);
413	if (defined $file) {
414	print "OK\n";
415	push @urls, [ $hostid, $url ];
416	} else {
417	print "REQUEST FAILURE\n";
418	}
419	} else {
420	print "skipping; status = $hosts->{$hostid}->{status}\n";
421	}
422	}
423
424	# everything should be chill
425	fail_text('no_devices') unless @urls;
426	print "\n";
427
428	# step 3: check devices for each host
429	print "Checking devices...\n";
430	printf " host device %10s %10s %10s %7s %7s %4s\n", 'size(G)', 'used(G)', 'free(G)', 'use% ', 'ob state', 'I/O%';
431	printf " ---- ------------ ---------- ---------- ---------- ------ ---------- -----\n";
432	my %total;
433	# Initialize to zero so that the total outputs doesn't need to check for undefined.
434	map { $total{$_} = 0; } qw(total used avail);
435
436	foreach my $hosturl (@urls) {
437	my ($hostid, $url) = @$hosturl;
438	my $devs = $devices->{$hostid};
439	foreach my $devid (sort { $a <=> $b } keys %$devs) {
440	my $dev = $devs->{$devid};
441	my $status = $dev->{status} \|\| "??";
442	next if $status eq "dead";
443
444	printf " [%2d] %-7s", $hostid, "dev$devid";
445
446	my $usage = get($url . "/dev$devid/usage");
447	if (defined $usage) {
448	my %data = ( map { split(/:\s+/, $_) } split(/\r?\n/, $usage) );
449	foreach (qw(time used total avail)) {
450	$data{$_} = 0 if (!$data{$_} \|\|
451	$data{$_} !~ /\A\d+(\.\d+)?\Z/);
452	}
453	$data{age} = $now - $data{time};
454	$data{used} /= 1024**2;
455	$data{total} /= 1024**2;
456	$data{avail} = $data{total} - $data{used};
457	my $pct = $data{used}/$data{total}*100;
458	$total{used} += $data{used};
459	$total{avail} += $data{avail};
460	$total{total} += $data{total};
461
462	$dev->{utilization} = 0 if (!defined($dev->{utilization}) \|\|
463	$dev->{utilization} !~ /\A\d+(\.\d+)?\Z/);
464	printf(" %10.3f %10.3f %10.3f %6.2f%% %-7s %5.1f\n",
465	(map { $data{$_} } qw(total used avail)),
466	$pct, ($dev->{observed_state} \|\| "?"),
467	$dev->{utilization});
468	} else {
469	print "REQUEST FAILURE\n";
470	}
471	}
472	}
473	my $pct = 0;
474	# Avoid division by zero
475	$pct = $total{used}/$total{total}*100 if($total{total} > 0);
476
477	printf " ---- ------------ ---------- ---------- ---------- ------\n";
478	printf " total:%10.3f %10.3f %10.3f %6.2f%%\n", (map { $total{$_} } qw(total used avail)), $pct;
479
480	# if we get here, all's well
481	ok();
482	}
483
484	sub cmd_stats {
485	print "Fetching statistics...\n";
486	my $stats = stats()
487	or fail("Can't fetch stats");
488
489	print "\nStatistics for devices...\n";
490	printf " %-10s %-10s %10s %10s\n", "device", "host", "files", "status";
491	printf " ---------- ----------- ---------- ----------\n";
492	foreach my $device (sort { $a <=> $b } keys %{$stats->{devices}}) {
493	my $value = $stats->{devices}->{$device};
494	printf " %-10s %-10s %10s %10s\n", "dev$device", $value->{host}, $value->{files}, $value->{status};
495	}
496	printf " ---------- ----------- ---------- ----------\n";
497
498	print "\nStatistics for file ids...\n";
499	printf " Max file id: %s\n", $stats->{fids}->{max} \|\| 'none';
500
501	print "\nStatistics for files...\n";
502	printf " %-20s %-10s %10s\n", 'domain', 'class', 'files';
503	printf " -------------------- ----------- ----------\n";
504	foreach my $domain (sort keys %{$stats->{files}}) {
505	my $classes = $stats->{files}->{$domain};
506	foreach my $class (sort keys %$classes) {
507	my $files = $classes->{$class};
508	printf " %-20s %-10s %10s\n", $domain, $class, $files;
509	}
510	}
511	printf " -------------------- ----------- ----------\n";
512
513	print "\nStatistics for replication...\n";
514	printf " %-20s %-10s %10s %10s\n", 'domain', 'class', 'devcount', 'files';
515	printf " -------------------- ----------- ---------- ----------\n";
516	foreach my $domain (sort keys %{$stats->{replication}}) {
517	my $classes = $stats->{replication}->{$domain};
518	foreach my $class (sort keys %$classes) {
519	my $devcounts = $classes->{$class};
520	foreach my $devcount (sort { $a <=> $b } keys %$devcounts) {
521	my $files = $devcounts->{$devcount};
522	printf " %-20s %-10s %10s %10s\n", $domain, $class, $devcount, $files;
523	}
524	}
525	}
526	printf " -------------------- ----------- ---------- ----------\n";
527	ok();
528	}
529
530	sub cmd_host_list {
531	my $hosts = hosts();
532	fail_text('no_hosts') unless $hosts;
533
534	foreach my $hostid (sort keys %$hosts) {
535	my $host = $hosts->{$hostid};
536	print "$host->{hostname} [$hostid]: $host->{status}\n";
537	my @data = (
538	'IP', "$host->{hostip}:$host->{http_port}",
539	'Alt IP', $host->{altip},
540	'Alt Mask', $host->{altmask},
541	'GET Port', $host->{http_get_port},
542	);
543	while (my ($k, $v) = splice(@data, 0, 2)) {
544	next unless $v;
545	printf " %-10s\%s\n", "$k:", $v;
546	}
547	print "\n";
548	}
549	ok();
550	}
551
552	sub cmd_host_add {
553	my $args = shift;
554
555	my $hosts = hosts_byname();
556	fail_text('no_hosts') unless $hosts;
557
558	my $name = delete $args->{hostname};
559	cmd_help_die("No hostname") unless $name;
560	fail('Host already exists.') if $hosts->{$name};
561
562	# make sure we have an ip
563	unless ($args->{ip}) {
564	my $addr = gethostbyname($name);
565	fail_text('host_add_no_ip') unless $addr;
566	$args->{ip} = inet_ntoa($addr);
567	}
568
569	# defaults
570	$args->{port} \|\|= 7500;
571	$args->{status} \|\|= 'down';
572
573	# FIXME: verify the status can't be 'alive' if we can't get to ip:port
574	# OR BETTER: also make default status the reachability of that ip:port
575
576	# now create the host
577	my $mogadm = mogadm();
578	$mogadm->create_host($name, $args);
579	if ($mogadm->err) {
580	fail("Failure creating host: " . $mogadm->errstr);
581	}
582
583	ok('Host has been created.');
584	}
585
586	sub cmd_host_modify {
587	my $args = shift;
588	my $name = delete $args->{hostname};
589
590	# FIXME: verify the status can't be 'alive' if we can't get to ip:port
591
592	# now modify the host
593	my $mogadm = mogadm();
594	$mogadm->update_host($name, $args);
595	if ($mogadm->err) {
596	fail("Failure modifying host: " . $mogadm->errstr);
597	}
598
599	ok('Host has been modified.');
600	}
601
602	sub cmd_host_delete {
603	my $args = shift;
604	my $name = delete $args->{hostname};
605
606	# now modify the host
607	my $mogadm = mogadm();
608	$mogadm->delete_host($name);
609	if ($mogadm->err) {
610	fail("Failure deleting host: " . $mogadm->errstr);
611	}
612
613	ok('Host has been deleted.');
614	}
615
616	sub cmd_host_mark {
617	my $args = shift;
618
619	my $mogadm = mogadm();
620	$mogadm->update_host($args->{hostname}, { status => $args->{status} });
621	if ($mogadm->err) {
622	fail("Failure updating host status: " . $mogadm->errstr);
623	}
624
625	ok('Host status updated.');
626	}
627
628	sub cmd_domain_list {
629	# actually lists domains and classes
630	my $domains = domains() or
631	fail_text('no_domains');
632
633	# now iterate
634	printf " %-20s %-20s \%s\n", "domain", "class", "mindevcount";
635	printf "%-20s %-20s \%s\n", '-' x 20, '-' x 20, '-' x 13;
636	foreach my $domain (sort keys %$domains) {
637	foreach my $class (sort keys %{$domains->{$domain}}) {
638	printf " %-20s %-20s %d\n", $domain, $class, $domains->{$domain}->{$class} \|\| 0;
639	}
640	print "\n";
641	}
642
643	ok();
644	}
645
646	sub cmd_domain_add {
647	my $args = shift;
648
649	my $domains = domains() or
650	fail_text('no_domains');
651
652	# make sure it doesn't exist
653	my $domain = delete $args->{domain};
654	fail('Domain already exists.') if $domains->{$domain};
655
656	# create
657	my $mogadm = mogadm();
658	$mogadm->create_domain($domain);
659	if ($mogadm->err) {
660	fail('Error creating domain: ' . $mogadm->errstr);
661	}
662
663	ok('Domain created.');
664	}
665
666	sub cmd_domain_delete {
667	my $args = shift;
668	my $domains = domains() or
669	fail_text('no_domains');
670
671	# make sure it doesn't exist
672	my $domain = $args->{domain};
673	fail('Domain not found.') unless $domains->{$domain};
674
675	# destroy
676	my $mogadm = mogadm();
677	$mogadm->delete_domain($domain);
678	if ($mogadm->err) {
679	fail('Error deleting domain: ' . $mogadm->errstr);
680	}
681
682	ok('Domain deleted.');
683	}
684
685	sub cmd_class_list {
686	# same, pass it through
687	cmd_domain_list();
688	}
689
690	sub cmd_class_add {
691	my $args = shift;
692	my $domains = domains() or
693	fail_text('no_domains');
694
695	my $domain = $args->{domain};
696	my $class = $args->{class};
697
698	cmd_help_die() unless $domain && $class;
699	fail('Domain not found.') unless $domains->{$domain};
700	fail('Class already exists.') if $domains->{$domain}->{$class};
701
702	$args->{mindevcount} \|\|= 2;
703
704	my $mogadm = mogadm();
705	$mogadm->create_class($domain, $class, $args->{mindevcount});
706	if ($mogadm->err) {
707	fail('Error creating class: ' . $mogadm->errstr);
708	}
709
710	ok('Class created.');
711	}
712
713	sub cmd_class_modify {
714	my $args = shift;
715	my $domains = domains() or
716	fail_text('no_domains');
717
718	my $domain = $args->{domain};
719	my $class = $args->{class};
720
721	cmd_help_die() unless $domain && $class;
722	fail('Domain not found.') unless $domains->{$domain};
723	fail('Class does not exist.') unless $domains->{$domain}->{$class};
724
725	$args->{mindevcount} \|\|= 2;
726
727	my $mogadm = mogadm();
728	$mogadm->update_class($domain, $class, $args->{mindevcount});
729	if ($mogadm->err) {
730	fail('Error updating class: ' . $mogadm->errstr);
731	}
732
733	ok('Class updated.');
734	}
735
736	sub cmd_class_delete {
737	my $args = shift;
738
739	my $domains = domains() or
740	fail_text('no_domains');
741
742	my $domain = $args->{domain};
743	my $class = $args->{class};
744
745	cmd_help_die() unless $domain && $class;
746	fail('Domain not found.') unless $domains->{$domain};
747	fail('Class does not exist.') unless $domains->{$domain}->{$class};
748
749	my $mogadm = mogadm();
750	$mogadm->delete_class($domain, $class);
751	if ($mogadm->err) {
752	fail('Error deleting class: ' . $mogadm->errstr);
753	}
754
755	ok('Class deleted.');
756	}
757
758	sub cmd_device_add {
759	my $args = shift;
760
761	my $hosts = hosts() or
762	fail_text('no_hosts');
763
764	my $host = $args->{hostname};
765	my $devid = $args->{devid};
766	my $state = $args->{status} \|\| "alive";
767
768	cmd_help_die("devid should be numeric") unless $devid =~ /^\d+$/;
769
770	# FIXME: server should be fixed to verify via HTTP that the devid directory exists
771
772	my $mogadm = mogadm();
773	$mogadm->create_device(hostname => $host, devid => $devid, state => $state);
774
775	if ($mogadm->err) {
776	fail('Error adding device: ' . $mogadm->errstr);
777	}
778
779	ok('Device added.');
780	}
781
782	sub cmd_device_mark {
783	my $args = shift;
784
785	my $mogadm = mogadm();
786	$mogadm->change_device_state($args->{hostname},
787	$args->{devid},
788	$args->{status});
789	if ($mogadm->err) {
790	fail('Error updating device: ' . $mogadm->errstr);
791	}
792
793	ok('Device updated.');
794	}
795
796	sub cmd_device_modify {
797	my $args = shift;
798	my $hostname = delete $args->{hostname};
799	my $devid = delete $args->{devid};
800
801	my $mogadm = mogadm();
802	$mogadm->update_device($hostname, $devid, $args);
803
804	if ($mogadm->err) {
805	fail('Error updating device: ' . $mogadm->errstr);
806	}
807
808	ok('Device updated.');
809	}
810
811	sub cmd_device_list {
812	my $args = shift;
813
814	my $hosts = hosts();
815	fail_text('no_hosts') unless $hosts;
816
817	my $devs = devices();
818	fail_text('no_devices') unless $devs;
819
820	foreach my $hostid (sort keys %$hosts) {
821	my $host = $hosts->{$hostid};
822	print "$host->{hostname} [$hostid]: $host->{status}\n";
823
824	printf "%6s %-10s %7s %7s %7s\n", '', '', 'used(G)', 'free(G)', 'total(G)';
825	foreach my $devid (sort keys %{$devs->{$hostid} \|\| {}}) {
826	my $dev = $devs->{$hostid}->{$devid};
827	next if $dev->{status} eq "dead" && ! $args->{all};
828
829	my $total = $dev->{mb_total} / 1024;
830	my $used = $dev->{mb_used} / 1024;
831	my $free = $total - $used;
832	printf "%6s: %-10s %-7.3f %-7.3f %-7.3f\n", "dev$devid", $dev->{status}, $used, $free, $total;
833	}
834
835	print "\n";
836	}
837
838	ok();
839	}
840
841	sub cmd_device_summary {
842	my $args = shift;
843	my %show_state;
844	$show_state{$_} = 1 foreach split(/,/, ($args->{status} \|\| "alive,readonly"));
845
846	my $hosts = hosts();
847	fail_text('no_hosts') unless $hosts;
848
849	my $devs = devices();
850	fail_text('no_devices') unless $devs;
851
852	printf "%-15s %6s %7s %8s %8s %8s\n", 'Hostname', 'HostID', 'Status', 'used(G)', 'free(G)', 'total(G)';
853	foreach my $hostid (sort keys %$hosts) {
854	my $host = $hosts->{$hostid};
855	my ($total,$used) = (0, 0);
856
857	foreach my $devid (sort keys %{$devs->{$hostid} \|\| {}}) {
858	my $dev = $devs->{$hostid}->{$devid};
859	next unless $show_state{$dev->{status}};
860
861	my $devtotal = $dev->{mb_total} / 1024;
862	my $devused = $dev->{mb_used} / 1024;
863
864	$total += $devtotal;
865	$used += $devused;
866	}
867	my $free = $total - $used;
868	printf "%-15s [%4d]: %6s", $host->{hostname}, $hostid, $host->{status};
869	printf " %8.3f %8.3f %8.3f\n", $used, $free, $total;
870	}
871
872	ok();
873
874	}
875
876	sub cmd_slave_list {
877	my $mogadm = mogadm();
878
879	my $slaves = $mogadm->slave_list();
880
881	foreach my $key (sort keys %$slaves) {
882	my $slavedata = $slaves->{$key};
883	my ($dsn, $username, $password) = @$slavedata;
884	print "$key --dsn=$dsn --username=$username --password=$password\n";
885	}
886
887	ok();
888	}
889
890	sub cmd_slave_add {
891	my $mogadm = mogadm();
892	my $args = shift;
893
894	my $rc = $mogadm->slave_add($args->{slave_key}, $args->{dsn}, $args->{username}, $args->{password});
895
896	if ($rc) {
897	ok("Slave added");
898	} else {
899	fail("Slave failed to be added");
900	}
901	}
902
903	sub cmd_slave_modify {
904	my $mogadm = mogadm();
905	my $args = shift;
906
907	my $key = delete $args->{slave_key} or cmd_help_die("Key argument is required");
908
909	my $rc = $mogadm->slave_modify($key, %$args);
910
911	if ($rc) {
912	ok("Slave modify success");
913	} else {
914	fail("Slave modify failure: " . $mogadm->errstr);
915	}
916	}
917
918	sub cmd_slave_delete {
919	my $mogadm = mogadm();
920
921	my $args = shift;
922
923	my $rc = $mogadm->slave_delete($args->{slave_key});
924
925	if ($rc) {
926	ok("Slave deleted");
927	} else {
928	fail("Slave delete failed");
929	}
930	}
931
932	sub cmd_fsck_start {
933	my $mogadm = mogadm();
934	my $res = $mogadm->fsck_start \|\| fail($mogadm->errstr);
935	ok("fsck started");
936	}
937
938	sub cmd_fsck_stop {
939	my $mogadm = mogadm();
940	my $res = $mogadm->fsck_stop \|\| fail($mogadm->errstr);
941	ok("fsck stopped");
942	}
943
944	sub cmd_fsck_reset {
945	my $mogadm = mogadm();
946	my $args = shift;
947	my $res = $mogadm->fsck_reset(
948	policy_only => $args->{"policy-only"},
949	startpos => $args->{"startpos"},
950	)
951	or fail($mogadm->errstr);
952	ok("fsck stopped");
953	}
954
955	sub cmd_fsck_clearlog {
956	my $mogadm = mogadm();
957	my $res = $mogadm->fsck_clearlog \|\| fail($mogadm->errstr);
958	ok("fsck log cleared");
959	}
960
961	sub _log_dump {
962	my %opts = @_;
963	my $max = $opts{start};
964	my $mogadm = mogadm();
965
966	my $fmt = "%-20s %5s %13s %10s\n";
967	printf($fmt, "unixtime", "event", "fid", "devid");
968	while (1) {
969	my @rows = $mogadm->fsck_log_rows(after_logid => $max);
970	unless (@rows) {
971	$opts{on_stall}->();
972	next;
973	}
974	foreach my $row (@rows) {
975	printf($fmt,
976	$row->{utime},
977	$row->{evcode},
978	$row->{fid},
979	$row->{devid} \|\| "-");
980	$max = $row->{logid};
981	}
982	}
983	}
984
985	sub cmd_fsck_printlog {
986	_log_dump(start => 0,
987	on_stall => sub { exit 0; });
988	}
989
990	sub cmd_fsck_taillog {
991	my $mogadm = mogadm();
992	my $status = $mogadm->fsck_status
993	or fail("can't get fsck status");
994	_log_dump(start => $status->{max_logid} - 20,
995	on_stall => sub { sleep 5; });
996	}
997
998	sub cmd_fsck_status {
999	my $mogadm = mogadm();
1000	my $status = $mogadm->fsck_status
1001	or fail("can't get fsck status");
1002
1003	my %known = map { $_ => 1 } qw(
1004	current_time
1005	max_logid
1006	);
1007	my $st = sub {
1008	my $k = shift;
1009	$known{$k} = 1;
1010	return $status->{$k};
1011	};
1012
1013	my $line = sub {
1014	printf("%11s: %-s\n", @_);
1015	};
1016	print "\n";
1017	my $host = $st->('host');
1018	$line->("Running", $st->('running') ? "Yes (on $host)" : "No");
1019
1020	my $ratio = $st->('end_fid') ? ($st->('max_fid_checked') / $st->('end_fid')) : 0;
1021	my $perc = sprintf("%0.02f%%", 100 * $ratio);
1022
1023	$line->("Status",
1024	$st->('max_fid_checked') . " / " . $st->('end_fid')
1025	. " ($perc)");
1026	my $elap = $st->('start_time') ?
1027	(($st->('stop_time') \|\| $st->('current_time')) - $st->('start_time')) :
1028	0;
1029	my $as_time = sub {
1030	my $s = shift;
1031	return int($s) . "s" if $s < 60;
1032	return int($s/60) . "m";
1033	};
1034	my $per_sec = $elap ? ($st->('max_fid_checked') / $elap) : 0;
1035	$line->("Time", sprintf("%s (%d fids/s; %s remain)",
1036	$as_time->($elap),
1037	sprintf("%0.1f", $per_sec),
1038	$as_time->($per_sec ?
1039	(($st->('end_fid') - $st->('max_fid_checked'))
1040	/ $per_sec) :
1041	0)));
1042
1043	$line->("Check Type", ($st->('policy_only') ?
1044	"Repl policy only (skip file checks)" :
1045	"Normal (check policy + files)"));
1046
1047	if (my @unk = grep { !$known{$_} } sort keys %$status) {
1048	print "\n";
1049	foreach (@unk) {
1050	$line->("[$_]", $status->{$_});
1051	}
1052	}
1053	print "\n";
1054	}
1055
1056	sub cmd_settings_list {
1057	my $mogadm = mogadm();
1058	unless ($mogadm->can("server_settings")) {
1059	fail("settings commands require MogileFS::Client >= 1.07");
1060	}
1061	my $ss = $mogadm->server_settings
1062	or fail("can't get settings");
1063	foreach my $k (sort keys %$ss) {
1064	printf("%25s = %-s\n", $k, $ss->{$k});
1065	}
1066	}
1067
1068	sub cmd_settings_set {
1069	my $mogadm = mogadm();
1070	unless ($mogadm->can("set_server_setting")) {
1071	fail("settings commands require MogileFS::Client >= 1.07");
1072	}
1073	my $args = shift;
1074
1075	$mogadm->set_server_setting($args->{key}, $args->{value})
1076	or fail($mogadm->errstr);
1077	ok();
1078	}
1079
1080	###########################################################################
1081	## helper routines
1082	###########################################################################
1083
1084	sub abortWithUsage {
1085	my $ret = "Usage: (enter any command prefix, leaving off options, for further help)\n\n";
1086	foreach my $cmd (@topcmds) {
1087	my $sbc = $usage->{$cmd}->{subcmd};
1088	if ($sbc) {
1089	$ret .= " mogadm $cmd ...\n";
1090	} else {
1091	$ret .= sprintf(" mogadm %-25s %-s\n",
1092	"$cmd",
1093	$usage->{$cmd}{des} \|\| "");
1094	next;
1095	}
1096	foreach my $v (sort keys %$sbc) {
1097	my $scv = $usage->{$cmd}{subcmd}{$v};
1098	$ret .= " ";
1099	my $dotdot = $scv->{args} ? "..." : "";
1100	$ret .= sprintf(" %-25s %-s\n",
1101	"$cmd $v $dotdot",
1102	$scv->{des} \|\| "");
1103
1104	}
1105	}
1106	print $ret, "\n";
1107	exit(1);
1108	}
1109
1110	sub abort_with_topcmd_help {
1111	my ($cmd, $verb, $msg) = @_;
1112	if ($msg) {
1113	print "\nERROR: $msg\n\n";
1114	}
1115	my $cmdsfx = $verb ? "-$verb" : "";
1116	&nbs