Index: /branches/release-40/extlib/I18N/LangTags/Detect.pm =================================================================== --- /branches/release-40/extlib/I18N/LangTags/Detect.pm (revision 2594) +++ /branches/release-40/extlib/I18N/LangTags/Detect.pm (revision 2594) @@ -0,0 +1,237 @@ + +# Time-stamp: "2004-06-20 21:47:55 ADT" + +require 5; +package I18N::LangTags::Detect; +use strict; + +use vars qw( @ISA $VERSION $MATCH_SUPERS $USING_LANGUAGE_TAGS + $USE_LITERALS $MATCH_SUPERS_TIGHTLY); + +BEGIN { unless(defined &DEBUG) { *DEBUG = sub () {0} } } + # define the constant 'DEBUG' at compile-time + +$VERSION = "1.03"; +@ISA = (); +use I18N::LangTags qw(alternate_language_tags locale2language_tag); + +sub _uniq { my %seen; return grep(!($seen{$_}++), @_); } +sub _normalize { + my(@languages) = + map lc($_), + grep $_, + map {; $_, alternate_language_tags($_) } @_; + return _uniq(@languages) if wantarray; + return $languages[0]; +} + +#--------------------------------------------------------------------------- +# The extent of our functional interface: + +sub detect () { return __PACKAGE__->ambient_langprefs; } + +#=========================================================================== + +sub ambient_langprefs { # always returns things untainted + my $base_class = $_[0]; + + return $base_class->http_accept_langs + if length( $ENV{'REQUEST_METHOD'} || '' ); # I'm a CGI + # it's off in its own routine because it's complicated + + # Not running as a CGI: try to puzzle out from the environment + my @languages; + + foreach my $envname (qw( LANGUAGE LC_ALL LC_MESSAGES LANG )) { + next unless $ENV{$envname}; + DEBUG and print "Noting \$$envname: $ENV{$envname}\n"; + push @languages, + map locale2language_tag($_), + # if it's a lg tag, fine, pass thru (untainted) + # if it's a locale ID, try converting to a lg tag (untainted), + # otherwise nix it. + + split m/[,:]/, + $ENV{$envname} + ; + last; # first one wins + } + + if($ENV{'IGNORE_WIN32_LOCALE'}) { + # no-op + } elsif(&_try_use('Win32::Locale')) { + # If we have that module installed... + push @languages, Win32::Locale::get_language() || '' + if defined &Win32::Locale::get_language; + } + return _normalize @languages; +} + +#--------------------------------------------------------------------------- + +sub http_accept_langs { + # Deal with HTTP "Accept-Language:" stuff. Hassle. + # This code is more lenient than RFC 3282, which you must read. + # Hm. Should I just move this into I18N::LangTags at some point? + no integer; + + my $in = (@_ > 1) ? $_[1] : $ENV{'HTTP_ACCEPT_LANGUAGE'}; + # (always ends up untainting) + + return() unless defined $in and length $in; + + $in =~ s/\([^\)]*\)//g; # nix just about any comment + + if( $in =~ m/^\s*([a-zA-Z][-a-zA-Z]+)\s*$/s ) { + # Very common case: just one language tag + return _normalize $1; + } elsif( $in =~ m/^\s*[a-zA-Z][-a-zA-Z]+(?:\s*,\s*[a-zA-Z][-a-zA-Z]+)*\s*$/s ) { + # Common case these days: just "foo, bar, baz" + return _normalize( $in =~ m/([a-zA-Z][-a-zA-Z]+)/g ); + } + + # Else it's complicated... + + $in =~ s/\s+//g; # Yes, we can just do without the WS! + my @in = $in =~ m/([^,]+)/g; + my %pref; + + my $q; + foreach my $tag (@in) { + next unless $tag =~ + m/^([a-zA-Z][-a-zA-Z]+) + (?: + ;q= + ( + \d* # a bit too broad of a RE, but so what. + (?: + \.\d+ + )? + ) + )? + $ + /sx + ; + $q = (defined $2 and length $2) ? $2 : 1; + #print "$1 with q=$q\n"; + push @{ $pref{$q} }, lc $1; + } + + return _normalize( + # Read off %pref, in descending key order... + map @{$pref{$_}}, + sort {$b <=> $a} + keys %pref + ); +} + +#=========================================================================== + +my %tried = (); + # memoization of whether we've used this module, or found it unusable. + +sub _try_use { # Basically a wrapper around "require Modulename" + # "Many men have tried..." "They tried and failed?" "They tried and died." + return $tried{$_[0]} if exists $tried{$_[0]}; # memoization + + my $module = $_[0]; # ASSUME sane module name! + { no strict 'refs'; + return($tried{$module} = 1) + if defined(%{$module . "::Lexicon"}) or defined(@{$module . "::ISA"}); + # weird case: we never use'd it, but there it is! + } + + print " About to use $module ...\n" if DEBUG; + { + local $SIG{'__DIE__'}; + eval "require $module"; # used to be "use $module", but no point in that. + } + if($@) { + print "Error using $module \: $@\n" if DEBUG > 1; + return $tried{$module} = 0; + } else { + print " OK, $module is used\n" if DEBUG; + return $tried{$module} = 1; + } +} + +#--------------------------------------------------------------------------- +1; +__END__ + + +=head1 NAME + +I18N::LangTags::Detect - detect the user's language preferences + +=head1 SYNOPSIS + + use I18N::LangTags::Detect; + my @user_wants = I18N::LangTags::Detect::detect(); + +=head1 DESCRIPTION + +It is a common problem to want to detect what language(s) the user would +prefer output in. + +=head1 FUNCTIONS + +This module defines one public function, +C. This function is not exported +(nor is even exportable), and it takes no parameters. + +In scalar context, the function returns the most preferred language +tag (or undef if no preference was seen). + +In list context (which is usually what you want), +the function returns a +(possibly empty) list of language tags representing (best first) what +languages the user apparently would accept output in. You will +probably want to pass the output of this through +C +or +C, like so: + + my @languages = + I18N::LangTags::implicate_supers_tightly( + I18N::LangTags::Detect::detect() + ); + + +=head1 ENVIRONMENT + +This module looks for several environment variables, including +REQUEST_METHOD, HTTP_ACCEPT_LANGUAGE, +LANGUAGE, LC_ALL, LC_MESSAGES, and LANG. + +It will also use the L module, if it's installed. + + +=head1 SEE ALSO + +L, L, L. + +(This module's core code started out as a routine in Locale::Maketext; +but I moved it here once I realized it was more generally useful.) + + +=head1 COPYRIGHT + +Copyright (c) 1998-2004 Sean M. Burke. All rights reserved. + +This library is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +The programs and documentation in this dist are distributed in +the hope that they will be useful, but without any warranty; without +even the implied warranty of merchantability or fitness for a +particular purpose. + + +=head1 AUTHOR + +Sean M. Burke C + +=cut + +# a tip: Put a bit of chopped up pickled ginger in your salad. It's tasty! Index: /branches/release-40/extlib/I18N/LangTags/List.pm =================================================================== --- /branches/release-40/extlib/I18N/LangTags/List.pm (revision 1098) +++ /branches/release-40/extlib/I18N/LangTags/List.pm (revision 2594) @@ -2,8 +2,8 @@ require 5; package I18N::LangTags::List; -# Time-stamp: "2002-02-02 20:13:58 MST" +# Time-stamp: "2004-10-06 23:26:21 ADT" use strict; -use vars qw(%Name $Debug $VERSION); -$VERSION = '0.25'; +use vars qw(%Name %Is_Disrec $Debug $VERSION); +$VERSION = '0.35'; # POD at the end. @@ -13,16 +13,21 @@ my $seeking = 1; my $count = 0; - my($tag,$name); + my($disrec,$tag,$name); + my $last_name = ''; while() { if($seeking) { $seeking = 0 if m/=for woohah/; - } else { - next unless ($tag, $name) = - m/\{([-0-9a-zA-Z]+)\}(?:\s*:)?\s*([^\[\]]+)/; + } elsif( ($disrec, $tag, $name) = + m/(\[?)\{([-0-9a-zA-Z]+)\}(?:\s*:)?\s*([^\[\]]+)/ + ) { $name =~ s/\s*[;\.]*\s*$//g; next unless $name; ++$count; print "<$tag> <$name>\n" if $Debug; - $Name{$tag} = $name; + $last_name = $Name{$tag} = $name; + $Is_Disrec{$tag} = 1 if $disrec; + } elsif (m/[Ff]ormerly \"([-a-z0-9]+)\"/) { + $Name{$1} = "$last_name (old tag)" if $last_name; + $Is_Disrec{$1} = 1; } } @@ -72,4 +77,40 @@ } +#-------------------------------------------------------------------------- + +sub is_decent { + my $tag = lc($_[0] || return 0); + #require I18N::LangTags; + + return 0 unless + $tag =~ + /^(?: # First subtag + [xi] | [a-z]{2,3} + ) + (?: # Subtags thereafter + - # separator + [a-z0-9]{1,8} # subtag + )* + $/xs; + + my @supers = (); + foreach my $bit (split('-', $tag)) { + push @supers, + scalar(@supers) ? ($supers[-1] . '-' . $bit) : $bit; + } + return 0 unless @supers; + shift @supers if $supers[0] =~ m<^(i|x|sgn)$>s; + return 0 unless @supers; + + foreach my $f ($tag, @supers) { + return 0 if $Is_Disrec{$f}; + return 2 if $Name{$f}; + # so that decent subforms of indecent tags are decent + } + return 2 if $Name{$tag}; # not only is it decent, it's known! + return 1; +} + +#-------------------------------------------------------------------------- 1; @@ -103,4 +144,11 @@ The function I18N::LangTags::List::name(...) is not exported. + +This module also provides a function +C )> that returns true iff +the language tag is syntactically valid and is for general use (like +"fr" or "fr-ca", below). That is, it returns false for tags that are +syntactically invalid and for tags, like "aus", that are listed in +brackets below. This function is not exported. The map of tags-to-names that it uses is accessable as @@ -196,4 +244,8 @@ =item {ada} : Adangme +=item {ady} : Adyghe + +eq Adygei + =item {aa} : Afar @@ -206,5 +258,7 @@ =item [{afa} : Afro-Asiatic (Other)] -=item {aka} : Akan +=item {ak} : Akan + +(Formerly "aka".) =item {akk} : Akkadian @@ -265,6 +319,12 @@ =item {hy} : Armenian +=item {an} : Aragonese + =item [{art} : Artificial (Other)] +=item {ast} : Asturian + +eq Bable. + =item {as} : Assamese @@ -277,5 +337,7 @@ =item [{map} : Austronesian (Other)] -=item {ava} : Avaric +=item {av} : Avaric + +(Formerly "ava".) =item {ae} : Avestan @@ -291,4 +353,9 @@ eq Azeri +Notable forms: +{az-Arab} Azerbaijani in Arabic script; +{az-Cyrl} Azerbaijani in Cyrillic script; +{az-Latn} Azerbaijani in Latin script. + =item {ban} : Balinese @@ -297,5 +364,7 @@ =item {bal} : Baluchi -=item {bam} : Bambara +=item {bm} : Bambara + +(Formerly "bam".) =item [{bai} : Bamileke languages] @@ -404,12 +473,14 @@ Many forms are mutually un-intelligible in spoken media. -Notable subforms: +Notable forms: +{zh-Hans} Chinese, in simplified script; +{zh-Hant} Chinese, in traditional script; +{zh-tw} Taiwan Chinese; {zh-cn} PRC Chinese; +{zh-sg} Singapore Chinese; +{zh-mo} Macau Chinese; {zh-hk} Hong Kong Chinese; -{zh-mo} Macau Chinese; -{zh-sg} Singapore Chinese; -{zh-tw} Taiwan Chinese; {zh-guoyu} Mandarin [Putonghua/Guoyu]; -{zh-hakka} Hakka [formerly i-hakka]; +{zh-hakka} Hakka [formerly "i-hakka"]; {zh-min} Hokkien; {zh-min-nan} Southern Hokkien; @@ -448,7 +519,7 @@ eq Corse. -=item {cre} : Cree - -NOT Creek! +=item {cr} : Cree + +NOT Creek! (Formerly "cre".) =item {mus} : Creek @@ -477,4 +548,6 @@ =item {da} : Danish + +=item {dar} : Dargwa =item {day} : Dayak @@ -492,5 +565,7 @@ =item {din} : Dinka -=item {div} : Divehi +=item {dv} : Divehi + +eq Maldivian. (Formerly "div".) =item {doi} : Dogri @@ -556,4 +631,8 @@ eq Anglo-Saxon. (Historical) +=item {i-enochian} : Enochian (Artificial) + +=item {myv} : Erzya + =item {eo} : Esperanto @@ -562,5 +641,7 @@ =item {et} : Estonian -=item {ewe} : Ewe +=item {ee} : Ewe + +(Formerly "ewe".) =item {ewo} : Ewondo @@ -604,5 +685,7 @@ =item {fur} : Friulian -=item {ful} : Fulah +=item {ff} : Fulah + +(Formerly "ful".) =item {gaa} : Ga @@ -616,5 +699,7 @@ eq Galician -=item {lug} : Ganda +=item {lg} : Ganda + +(Formerly "lug".) =item {gay} : Gayo @@ -680,4 +765,8 @@ =item {hai} : Haida +=item {ht} : Haitian + +eq Haitian Creole + =item {ha} : Hausa @@ -717,5 +806,11 @@ =item {is} : Icelandic -=item {ibo} : Igbo +=item {io} : Ido + +(Artificial) + +=item {ig} : Igbo + +(Formerly "ibo".) =item {ijo} : Ijo @@ -733,4 +828,6 @@ =for etc {in} Indonesian (old tag) + +=item {inh} : Ingush =item {ia} : Interlingua (International Auxiliary Language Association) @@ -774,5 +871,7 @@ (NOT "jp"!) -=item {jw} : Javanese +=item {jv} : Javanese + +(Formerly "jw" because of a typo.) =item {jrb} : Judeo-Arabic @@ -780,4 +879,6 @@ =item {jpr} : Judeo-Persian +=item {kbd} : Kabardian + =item {kab} : Kabyle @@ -788,4 +889,6 @@ eq Greenlandic "Eskimo" +=item {xal} : Kalmyk + =item {kam} : Kamba @@ -794,5 +897,9 @@ eq Kanarese. NOT Canadian! -=item {kau} : Kanuri +=item {kr} : Kanuri + +(Formerly "kau".) + +=item {krc} : Karachay-Balkar =item {kaa} : Kara-Kalpak @@ -802,4 +909,8 @@ =item {ks} : Kashmiri +=item {csb} : Kashubian + +eq Kashub + =item {kaw} : Kawi @@ -830,5 +941,7 @@ =item {kv} : Komi -=item {kon} : Kongo +=item {kg} : Kongo + +(Formerly "kon".) =item {kok} : Konkani @@ -878,5 +991,5 @@ =item {lb} : Letzeburgesch -eq Luxemburgian, eq Luxemburger. (Formerly i-lux.) +eq Luxemburgian, eq Luxemburger. (Formerly "i-lux".) =for etc @@ -885,4 +998,8 @@ =item {lez} : Lezghian +=item {li} : Limburgish + +eq Limburger, eq Limburgan. NOT Letzeburgesch! + =item {ln} : Lingala @@ -893,7 +1010,11 @@ eq Low Saxon. eq Low German. eq Low Saxon. +=item {art-lojban} : Lojban (Artificial) + =item {loz} : Lozi -=item {lub} : Luba-Katanga +=item {lu} : Luba-Katanga + +(Formerly "lub".) =item {lua} : Luba-Lulua @@ -986,4 +1107,6 @@ =item {moh} : Mohawk +=item {mdf} : Moksha + =item {mo} : Moldavian @@ -1008,9 +1131,11 @@ =item {nah} : Nahuatl +=item {nap} : Neapolitan + =item {na} : Nauru =item {nv} : Navajo -eq Navaho. (Formerly i-navajo.) +eq Navaho. (Formerly "i-navajo".) =for etc @@ -1039,4 +1164,6 @@ =item {niu} : Niuean +=item {nog} : Nogai + =item {non} : Old Norse @@ -1047,8 +1174,4 @@ Do not use this. -=item {se} : Northern Sami - -eq Lappish. eq Lapp. eq (Northern) Saami. - =item {no} : Norwegian @@ -1057,5 +1180,5 @@ =item {nb} : Norwegian Bokmal -eq BokmEl, (A form of Norwegian.) (Formerly no-bok.) +eq BokmEl, (A form of Norwegian.) (Formerly "no-bok".) =for etc @@ -1064,5 +1187,5 @@ =item {nn} : Norwegian Nynorsk -(A form of Norwegian.) (Formerly no-nyn.) +(A form of Norwegian.) (Formerly "no-nyn".) =for etc @@ -1083,7 +1206,7 @@ eq ProvenEal, eq Provencal -=item {oji} : Ojibwa - -eq Ojibwe. +=item {oj} : Ojibwa + +eq Ojibwe. (Formerly "oji".) =item {or} : Oriya @@ -1203,4 +1326,16 @@ NOT Aramaic! +=item {se} : Northern Sami + +eq Lappish. eq Lapp. eq (Northern) Saami. + +=item {sma} : Southern Sami + +=item {smn} : Inari Sami + +=item {smj} : Lule Sami + +=item {sms} : Skolt Sami + =item [{smi} : Sami languages (Other)] @@ -1234,4 +1369,8 @@ eq Serb. NOT Sorbian. + +Notable forms: +{sr-Cyrl} : Serbian in Cyrillic script; +{sr-Latn} : Serbian in Latin script. =item {srr} : Serer @@ -1250,4 +1389,6 @@ {sgn-ni} Nicaraguan Sign Language (ISN); {sgn-us} American Sign Language (ASL). + +(And so on with other country codes as the subtag.) =item {bla} : Siksika @@ -1423,4 +1564,6 @@ =item {tum} : Tumbuka +=item [{tup} : Tupi languages] + =item {tr} : Turkish @@ -1431,4 +1574,8 @@ (Typically in Arabic script) (Historical) +=item {crh} : Crimean Turkish + +eq Crimean Tatar + =item {tk} : Turkmen @@ -1443,4 +1590,6 @@ =item {tw} : Twi +=item {udm} : Udmurt + =item {uga} : Ugaritic @@ -1463,9 +1612,13 @@ eq Ezbek +Notable forms: +{uz-Cyrl} Uzbek in Cyrillic script; +{uz-Latn} Uzbek in Latin script. + =item {vai} : Vai -=item {ven} : Venda - -NOT Wendish! NOT Wend! NOT Avestan! +=item {ve} : Venda + +NOT Wendish! NOT Wend! NOT Avestan! (Formerly "ven".) =item {vi} : Vietnamese @@ -1482,4 +1635,6 @@ =item [{wak} : Wakashan languages] + +=item {wa} : Walloon =item {wal} : Walamo @@ -1518,10 +1673,12 @@ eq Yap +=item {ii} : Sichuan Yi + =item {yi} : Yiddish -Formerly "ji". Sometimes in Roman script, sometimes in Hebrew script. - -=for etc -{ji} Yiddish (old tag) +Formerly "ji". Usually in Hebrew script. + +Notable forms: +{yi-latn} Yiddish in Latin script =item {yo} : Yoruba @@ -1559,5 +1716,5 @@ =head1 COPYRIGHT AND DISCLAIMER -Copyright (c) 2001,2002 Sean M. Burke. All rights reserved. +Copyright (c) 2001+ Sean M. Burke. All rights reserved. You can redistribute and/or Index: /branches/release-40/extlib/I18N/LangTags.pm =================================================================== --- /branches/release-40/extlib/I18N/LangTags.pm (revision 1098) +++ /branches/release-40/extlib/I18N/LangTags.pm (revision 2594) @@ -1,4 +1,4 @@ -# Time-stamp: "2002-02-02 20:43:03 MST" +# Time-stamp: "2004-10-06 23:26:33 ADT" # Sean M. Burke @@ -15,8 +15,13 @@ locale2language_tag alternate_language_tags encode_language_tag panic_languages + implicate_supers + implicate_supers_strictly ); %EXPORT_TAGS = ('ALL' => \@EXPORT_OK); -$VERSION = "0.27"; +$VERSION = "0.35"; + +sub uniq { my %seen; return grep(!($seen{$_}++), @_); } # a util function + =head1 NAME @@ -26,14 +31,13 @@ =head1 SYNOPSIS - use I18N::LangTags qw(is_language_tag same_language_tag - extract_language_tags super_languages - similarity_language_tag is_dialect_of - locale2language_tag alternate_language_tags - encode_language_tag panic_languages - ); - -...or whatever of those functions you want to import. Those are -all the exportable functions -- you're free to import only some, -or none at all. By default, none are imported. If you say: + use I18N::LangTags(); + +...or specify whichever of those functions you want to import, like so: + + use I18N::LangTags qw(implicate_supers similarity_language_tag); + +All the exportable functions are listed below -- you're free to import +only some, or none at all. By default, none are imported. If you +say: use I18N::LangTags qw(:ALL) @@ -400,5 +404,6 @@ $lang =~ tr<_><->; # "en_US" -> en-US - $lang =~ s<\.[-_a-zA-Z0-9\.]*><>s; # "en_US.ISO8859-1" -> en-US + $lang =~ s<(?:[\.\@][-_a-zA-Z0-9]+)+$><>s; # "en_US.ISO8859-1" -> en-US + # it_IT.utf8@euro => it-IT return $lang if &is_language_tag($lang); @@ -530,7 +535,14 @@ $tag =~ s/^iw\b/he/i; # Hebrew $tag =~ s/^in\b/id/i; # Indonesian + $tag =~ s/^cre\b/cr/i; # Cree + $tag =~ s/^jw\b/jv/i; # Javanese $tag =~ s/^[ix]-lux\b/lb/i; # Luxemburger $tag =~ s/^[ix]-navajo\b/nv/i; # Navajo $tag =~ s/^ji\b/yi/i; # Yiddish + # SMB 2003 -- Hm. There's a bunch of new XXX->YY variances now, + # but maybe they're all so obscure I can ignore them. "Obscure" + # meaning either that the language is obscure, and/or that the + # XXX form was extant so briefly that it's unlikely it was ever + # used. I hope. # # These go FROM the simplex to complex form, to get @@ -731,4 +743,82 @@ } +#--------------------------------------------------------------------------- +#--------------------------------------------------------------------------- + +=item * the function implicate_supers( ...languages... ) + +This takes a list of strings (which are presumed to be language-tags; +strings that aren't, are ignored); and after each one, this function +inserts super-ordinate forms that don't already appear in the list. +The original list, plus these insertions, is returned. + +In other words, it takes this: + + pt-br de-DE en-US fr pt-br-janeiro + +and returns this: + + pt-br pt de-DE de en-US en fr pt-br-janeiro + +This function is most useful in the idiom + + implicate_supers( I18N::LangTags::Detect::detect() ); + +(See L.) + + +=item * the function implicate_supers_strictly( ...languages... ) + +This works like C except that the implicated +forms are added to the end of the return list. + +In other words, implicate_supers_strictly takes a list of strings +(which are presumed to be language-tags; strings that aren't, are +ignored) and after the whole given list, it inserts the super-ordinate forms +of all given tags, minus any tags that already appear in the input list. + +In other words, it takes this: + + pt-br de-DE en-US fr pt-br-janeiro + +and returns this: + + pt-br de-DE en-US fr pt-br-janeiro pt de en + +The reason this function has "_strictly" in its name is that when +you're processing an Accept-Language list according to the RFCs, if +you interpret the RFCs quite strictly, then you would use +implicate_supers_strictly, but for normal use (i.e., common-sense use, +as far as I'm concerned) you'd use implicate_supers. + +=cut + +sub implicate_supers { + my @languages = grep is_language_tag($_), @_; + my %seen_encoded; + foreach my $lang (@languages) { + $seen_encoded{ I18N::LangTags::encode_language_tag($lang) } = 1 + } + + my(@output_languages); + foreach my $lang (@languages) { + push @output_languages, $lang; + foreach my $s ( I18N::LangTags::super_languages($lang) ) { + # Note that super_languages returns the longest first. + last if $seen_encoded{ I18N::LangTags::encode_language_tag($s) }; + push @output_languages, $s; + } + } + return uniq( @output_languages ); + +} + +sub implicate_supers_strictly { + my @tags = grep is_language_tag($_), @_; + return uniq( @_, map super_languages($_), @_ ); +} + + + ########################################################################### 1; @@ -771,17 +861,14 @@ C -* ISO 639, "Code for the representation of names of languages", -C - * ISO 639-2, "Codes for the representation of names of languages", -including three-letter codes, -C +including two-letter and three-letter codes, +C * The IANA list of registered languages (hopefully up-to-date), -C +C =head1 COPYRIGHT -Copyright (c) 1998-2001 Sean M. Burke. All rights reserved. +Copyright (c) 1998+ Sean M. Burke. All rights reserved. This library is free software; you can redistribute it and/or Index: /branches/release-40/extlib/Locale/Maketext/TPJ13.pod =================================================================== --- /branches/release-40/extlib/Locale/Maketext/TPJ13.pod (revision 1098) +++ /branches/release-40/extlib/Locale/Maketext/TPJ13.pod (revision 2594) @@ -1,3 +1,2 @@ - # This document contains text in Perl "POD" format. # Use a POD viewer like perldoc or perlman to render it. @@ -14,6 +13,6 @@ The following article by Sean M. Burke and Jordan Lachler -first appeared in I #13 and is copyright 1999 The Perl Journal. It appears +first appeared in I #13 +and is copyright 1999 The Perl Journal. It appears courtesy of Jon Orwant and The Perl Journal. This document may be distributed under the same terms as Perl itself. @@ -49,5 +48,5 @@ Your query matched 10 files in 4 directories. -So how hard could that be? You look at the code that produces +So how hard could that be? You look at the code that produces the first item, and it reads: @@ -82,5 +81,5 @@ $dir_scan_count, $dir_scan_count == 1 ? - gettext("directory") : gettext("directory"), + gettext("directory") : gettext("directories"), ); @@ -150,6 +149,6 @@ Chinese guy replies with the one phrase that these all translate to in Chinese, and that phrase has two "%g"s in it, as it should -- but -there's a problem. He translates it word-for-word back: "To your -question, in %g directories you would find %g answers." The "%g" +there's a problem. He translates it word-for-word back: "In %g +directories contains %g files match your query." The %g slots are in an order reverse to what they are in English. You wonder how you'll get gettext to handle that. @@ -207,5 +206,5 @@ He elaborates: In "I scanned %g directories", you'd I "directories" to be in the accusative case (since it is the direct -object in the sentnce) and the plural number, +object in the sentence) and the plural number, except where $directory_count is 1, then you'd expect the singular, of course. Just like Latin or German. I Where $directory_count % @@ -223,5 +222,5 @@ noun, when preceded by a number and in the nominative or accusative cases (as it is here, just your luck!), it does stay plural, but it is -forced into the genitive case -- yet another another ending... And +forced into the genitive case -- yet another ending... And you never hear him get to the part about how you're going to run into similar (but maybe subtly different) problems with other Slavic Index: /branches/release-40/extlib/Locale/Maketext/GutsLoader.pm =================================================================== --- /branches/release-40/extlib/Locale/Maketext/GutsLoader.pm (revision 2594) +++ /branches/release-40/extlib/Locale/Maketext/GutsLoader.pm (revision 2594) @@ -0,0 +1,49 @@ +package Locale::Maketext::GutsLoader; + +$VERSION = '1.13'; + +use strict; +sub zorp { return scalar @_ } + +BEGIN { + $Locale::Maketext::GutsLoader::GUTSPATH = __FILE__; + *Locale::Maketext::DEBUG = sub () {0} + unless defined &Locale::Maketext::DEBUG; +} + +# +# This whole drama is so that we can load the utf8'd code +# in Locale::Maketext::Guts, but if that fails, snip the +# utf8 and then try THAT. +# + +$Locale::Maketext::GUTSPATH = ''; +Locale::Maketext::DEBUG and warn "Requiring Locale::Maketext::Guts...\n"; +eval 'require Locale::Maketext::Guts'; + +if ($@) { + my $path = $Locale::Maketext::GUTSPATH; + + die "Can't load Locale::Maketext::Guts\nAborting" unless $path; + + die "No readable file $Locale::Maketext::GutsLoader::GUTSPATH\nAborting" + unless -e $path and -f _ and -r _; + + open(IN, $path) or die "Can't read-open $path\nAborting"; + + my $source; + { local $/; $source = ; } + close(IN); + unless( $source =~ s/\b(use utf8)/# $1/ ) { + Locale::Maketext::DEBUG and + print "I didn't see 'use utf8' in $path\n"; + } + eval $source; + die "Can't compile $path\n...The error I got was:\n$@\nAborting" if $@; + Locale::Maketext::DEBUG and warn "Non-utf8'd Locale::Maketext::Guts fine\n"; +} +else { + Locale::Maketext::DEBUG and warn "Loaded Locale::Maketext::Guts fine\n"; +} + +1; Index: /branches/release-40/extlib/Locale/Maketext/Guts.pm =================================================================== --- /branches/release-40/extlib/Locale/Maketext/Guts.pm (revision 2594) +++ /branches/release-40/extlib/Locale/Maketext/Guts.pm (revision 2594) @@ -0,0 +1,328 @@ +package Locale::Maketext::Guts; + +$VERSION = '1.13'; + +BEGIN { + # Just so we're nice and define SOMETHING in "our" package. + *zorp = sub { return scalar @_ } unless defined &zorp; +} + +package Locale::Maketext; +use strict; +use vars qw($USE_LITERALS $GUTSPATH); + +BEGIN { + $GUTSPATH = __FILE__; + *DEBUG = sub () {0} unless defined &DEBUG; +} + +use utf8; + +sub _compile { + # This big scary routine compiles an entry. + # It returns either a coderef if there's brackety bits in this, or + # otherwise a ref to a scalar. + + my $target = ref($_[0]) || $_[0]; + + my(@code); + my(@c) = (''); # "chunks" -- scratch. + my $call_count = 0; + my $big_pile = ''; + { + my $in_group = 0; # start out outside a group + my($m, @params); # scratch + + while($_[1] =~ # Iterate over chunks. + m/\G( + [^\~\[\]]+ # non-~[] stuff + | + ~. # ~[, ~], ~~, ~other + | + \[ # [ presumably opening a group + | + \] # ] presumably closing a group + | + ~ # terminal ~ ? + | + $ + )/xgs + ) { + DEBUG>2 and print qq{ "$1"\n}; + + if($1 eq '[' or $1 eq '') { # "[" or end + # Whether this is "[" or end, force processing of any + # preceding literal. + if($in_group) { + if($1 eq '') { + $target->_die_pointing($_[1], 'Unterminated bracket group'); + } + else { + $target->_die_pointing($_[1], 'You can\'t nest bracket groups'); + } + } + else { + if ($1 eq '') { + DEBUG>2 and print " [end-string]\n"; + } + else { + $in_group = 1; + } + die "How come \@c is empty?? in <$_[1]>" unless @c; # sanity + if(length $c[-1]) { + # Now actually processing the preceding literal + $big_pile .= $c[-1]; + if($USE_LITERALS and ( + (ord('A') == 65) + ? $c[-1] !~ m/[^\x20-\x7E]/s + # ASCII very safe chars + : $c[-1] !~ m/[^ !"\#\$%&'()*+,\-.\/0-9:;<=>?\@A-Z[\\\]^_`a-z{|}~\x07]/s + # EBCDIC very safe chars + )) { + # normal case -- all very safe chars + $c[-1] =~ s/'/\\'/g; + push @code, q{ '} . $c[-1] . "',\n"; + $c[-1] = ''; # reuse this slot + } + else { + push @code, ' $c[' . $#c . "],\n"; + push @c, ''; # new chunk + } + } + # else just ignore the empty string. + } + + } + elsif($1 eq ']') { # "]" + # close group -- go back in-band + if($in_group) { + $in_group = 0; + + DEBUG>2 and print " --Closing group [$c[-1]]\n"; + + # And now process the group... + + if(!length($c[-1]) or $c[-1] =~ m/^\s+$/s) { + DEBUG > 2 and print " -- (Ignoring)\n"; + $c[-1] = ''; # reset out chink + next; + } + + #$c[-1] =~ s/^\s+//s; + #$c[-1] =~ s/\s+$//s; + ($m,@params) = split(/,/, $c[-1], -1); # was /\s*,\s*/ + + # A bit of a hack -- we've turned "~,"'s into DELs, so turn + # 'em into real commas here. + if (ord('A') == 65) { # ASCII, etc + foreach($m, @params) { tr/\x7F/,/ } + } + else { # EBCDIC (1047, 0037, POSIX-BC) + # Thanks to Peter Prymmer for the EBCDIC handling + foreach($m, @params) { tr/\x07/,/ } + } + + # Special-case handling of some method names: + if($m eq '_*' or $m =~ m/^_(-?\d+)$/s) { + # Treat [_1,...] as [,_1,...], etc. + unshift @params, $m; + $m = ''; + } + elsif($m eq '*') { + $m = 'quant'; # "*" for "times": "4 cars" is 4 times "cars" + } + elsif($m eq '#') { + $m = 'numf'; # "#" for "number": [#,_1] for "the number _1" + } + + # Most common case: a simple, legal-looking method name + if($m eq '') { + # 0-length method name means to just interpolate: + push @code, ' ('; + } + elsif($m =~ /^\w+(?:\:\:\w+)*$/s + and $m !~ m/(?:^|\:)\d/s + # exclude starting a (sub)package or symbol with a digit + ) { + # Yes, it even supports the demented (and undocumented?) + # $obj->Foo::bar(...) syntax. + $target->_die_pointing( + $_[1], q{Can't use "SUPER::" in a bracket-group method}, + 2 + length($c[-1]) + ) + if $m =~ m/^SUPER::/s; + # Because for SUPER:: to work, we'd have to compile this into + # the right package, and that seems just not worth the bother, + # unless someone convinces me otherwise. + + push @code, ' $_[0]->' . $m . '('; + } + else { + # TODO: implement something? or just too icky to consider? + $target->_die_pointing( + $_[1], + "Can't use \"$m\" as a method name in bracket group", + 2 + length($c[-1]) + ); + } + + pop @c; # we don't need that chunk anymore + ++$call_count; + + foreach my $p (@params) { + if($p eq '_*') { + # Meaning: all parameters except $_[0] + $code[-1] .= ' @_[1 .. $#_], '; + # and yes, that does the right thing for all @_ < 3 + } + elsif($p =~ m/^_(-?\d+)$/s) { + # _3 meaning $_[3] + $code[-1] .= '$_[' . (0 + $1) . '], '; + } + elsif($USE_LITERALS and ( + (ord('A') == 65) + ? $p !~ m/[^\x20-\x7E]/s + # ASCII very safe chars + : $p !~ m/[^ !"\#\$%&'()*+,\-.\/0-9:;<=>?\@A-Z[\\\]^_`a-z{|}~\x07]/s + # EBCDIC very safe chars + )) { + # Normal case: a literal containing only safe characters + $p =~ s/'/\\'/g; + $code[-1] .= q{'} . $p . q{', }; + } + else { + # Stow it on the chunk-stack, and just refer to that. + push @c, $p; + push @code, ' $c[' . $#c . '], '; + } + } + $code[-1] .= "),\n"; + + push @c, ''; + } + else { + $target->_die_pointing($_[1], q{Unbalanced ']'}); + } + + } + elsif(substr($1,0,1) ne '~') { + # it's stuff not containing "~" or "[" or "]" + # i.e., a literal blob + $c[-1] .= $1; + + } + elsif($1 eq '~~') { # "~~" + $c[-1] .= '~'; + + } + elsif($1 eq '~[') { # "~[" + $c[-1] .= '['; + + } + elsif($1 eq '~]') { # "~]" + $c[-1] .= ']'; + + } + elsif($1 eq '~,') { # "~," + if($in_group) { + # This is a hack, based on the assumption that no-one will actually + # want a DEL inside a bracket group. Let's hope that's it's true. + if (ord('A') == 65) { # ASCII etc + $c[-1] .= "\x7F"; + } + else { # EBCDIC (cp 1047, 0037, POSIX-BC) + $c[-1] .= "\x07"; + } + } + else { + $c[-1] .= '~,'; + } + + } + elsif($1 eq '~') { # possible only at string-end, it seems. + $c[-1] .= '~'; + + } + else { + # It's a "~X" where X is not a special character. + # Consider it a literal ~ and X. + $c[-1] .= $1; + } + } + } + + if($call_count) { + undef $big_pile; # Well, nevermind that. + } + else { + # It's all literals! Ahwell, that can happen. + # So don't bother with the eval. Return a SCALAR reference. + return \$big_pile; + } + + die q{Last chunk isn't null??} if @c and length $c[-1]; # sanity + DEBUG and warn scalar(@c), " chunks under closure\n"; + if(@code == 0) { # not possible? + DEBUG and warn "Empty code\n"; + return \''; + } + elsif(@code > 1) { # most cases, presumably! + unshift @code, "join '',\n"; + } + unshift @code, "use strict; sub {\n"; + push @code, "}\n"; + + DEBUG and warn @code; + my $sub = eval(join '', @code); + die "$@ while evalling" . join('', @code) if $@; # Should be impossible. + return $sub; +} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +sub _die_pointing { + # This is used by _compile to throw a fatal error + my $target = shift; # class name + # ...leaving $_[0] the error-causing text, and $_[1] the error message + + my $i = index($_[0], "\n"); + + my $pointy; + my $pos = pos($_[0]) - (defined($_[2]) ? $_[2] : 0) - 1; + if($pos < 1) { + $pointy = "^=== near there\n"; + } + else { # we need to space over + my $first_tab = index($_[0], "\t"); + if($pos > 2 and ( -1 == $first_tab or $first_tab > pos($_[0]))) { + # No tabs, or the first tab is harmlessly after where we will point to, + # AND we're far enough from the margin that we can draw a proper arrow. + $pointy = ('=' x $pos) . "^ near there\n"; + } + else { + # tabs screw everything up! + $pointy = substr($_[0],0,$pos); + $pointy =~ tr/\t //cd; + # make everything into whitespace, but preseving tabs + $pointy .= "^=== near there\n"; + } + } + + my $errmsg = "$_[1], in\:\n$_[0]"; + + if($i == -1) { + # No newline. + $errmsg .= "\n" . $pointy; + } + elsif($i == (length($_[0]) - 1) ) { + # Already has a newline at end. + $errmsg .= $pointy; + } + else { + # don't bother with the pointy bit, I guess. + } + Carp::croak( "$errmsg via $target, as used" ); +} + +1; + Index: /branches/release-40/extlib/Locale/Maketext.pod =================================================================== --- /branches/release-40/extlib/Locale/Maketext.pod (revision 1098) +++ /branches/release-40/extlib/Locale/Maketext.pod (revision 2594) @@ -1,8 +1,8 @@ -# Time-stamp: "2001-06-21 23:12:39 MDT" +# Time-stamp: "2004-01-11 18:35:34 AST" =head1 NAME -Locale::Maketext -- framework for localization +Locale::Maketext - framework for localization =head1 SYNOPSIS @@ -119,5 +119,5 @@ that succeeds, returns YourProjClass::I->new(). -It runs thru the entire given list of language-tags, and finds no classes +If it runs thru the entire given list of language-tags, and finds no classes for those exact terms, it then tries "superordinate" language classes. So if no "en-US" class (i.e., YourProjClass::en_us) @@ -147,5 +147,5 @@ Otherwise (i.e., if not a CGI), this tries various OS-specific ways to get the language-tags for the current locale/language, and then -pretends that those were the value(s) passed to C. +pretends that those were the value(s) passed to C. Currently this OS-specific stuff consists of looking in the environment @@ -163,7 +163,7 @@ sub get_handle_via_config { my $class = $_[0]; - my $preferred_language = $Config_settings{'language'}; + my $chosen_language = $Config_settings{'language'}; my $lh; - if($preferred_language) { + if($chosen_language) { $lh = $class->get_handle($chosen_language) || die "No language handle for \"$chosen_language\" or the like"; @@ -232,5 +232,5 @@ This is the most important method in Locale::Maketext: -$text = $lh->maketext(I, ...parameters for this phrase...); + $text = $lh->maketext(I, ...parameters for this phrase...); This looks in the %Lexicon of the language handle @@ -242,7 +242,10 @@ If the value is a scalarref, the scalar is dereferenced and returned (and any parameters are ignored). + If the value is a coderef, we return &$value($lh, ...parameters...). + If the value is a string that I look like it's in Bracket Notation, we return it (after replacing it with a scalarref, in its %Lexicon). + If the value I look like it's in Bracket Notation, then we compile it into a sub, replace the string in the %Lexicon with the new coderef, @@ -326,5 +329,5 @@ It's for I a noun (i.e., saying how much of it there is, -while giving the currect form of it). The behavior of this method is +while giving the correct form of it). The behavior of this method is handy for English and a few other Western European languages, and you should override it for languages where it's not suitable. You can feel @@ -348,5 +351,5 @@ for 1 it returns "1 file", and for more it returns "2 files", etc.) -But for "directory", you'd want C<"[quant,_1,direcory,directories]"> +But for "directory", you'd want C<"[quant,_1,directory,directories]"> so that our elementary C method doesn't think that the plural of "directory" is "directorys". And you might find that the @@ -449,5 +452,5 @@ the call to YourProjClass->get_handle(...). It should derive (whether directly or indirectly) from Locale::Maketext. -It B how you name this class, altho assuming this +It B how you name this class, although assuming this is the localization component of your Super Mega Program, good names for your project class might be @@ -461,5 +464,5 @@ It will look for them by taking each language-tag (B it if it doesn't look like a language-tag or locale-tag!), turning it to -all lowercase, turning and dashes to underscores, and appending it +all lowercase, turning dashes to underscores, and appending it to YourProjClass . "::". So this: @@ -484,5 +487,5 @@ =item * -Language classes may derive from other language classes (altho they +Language classes may derive from other language classes (although they should have "use I" or "use base qw(I<...classes...>)"). They may derive from the project @@ -512,5 +515,5 @@ While the key must be a string value (since that's a basic restriction that Perl places on hash keys), the value in -the lexicon can currenly be of several types: +the lexicon can currently be of several types: a defined scalar, scalarref, or coderef. The use of these is explained above, in the section 'The "maketext" Method', and @@ -567,9 +570,9 @@ valid lexicon values. One notable exception is when the value is quite long. For example, to get the screenful of data that -a command-line program might returns when given an unknown switch, -I often just use a key "_USAGE_MESSAGE". At that point I then go +a command-line program might return when given an unknown switch, +I often just use a brief, self-explanatory key such as "_USAGE_MESSAGE". At that point I then go and immediately to define that lexicon entry in the ProjectClass::L10N::en lexicon (since English is always my "project -lanuage"): +language"): '_USAGE_MESSAGE' => <<'EOSTUFF', @@ -586,5 +589,5 @@ special hashes I, but because you access them via the C method, which looks for entries across all the -C<%Lexicon>'s in a language class I all its ancestor classes. +C<%Lexicon> hashes in a language class I all its ancestor classes. (This is because the idea of "class data" isn't directly implemented in Perl, but is instead left to individual class-systems to implement @@ -594,5 +597,5 @@ besides just phrases for output: for example, if your program takes input from the keyboard, asking a "(Y/N)" question, -you probably need to know what equivalent of "Y[es]/N[o]" is +you probably need to know what the equivalent of "Y[es]/N[o]" is in whatever language. You probably also need to know what the equivalents of the answers "y" and "n" are. You can @@ -605,5 +608,5 @@ Or instead of storing this in the language class's lexicon, you can (and, in some cases, really should) represent the same bit -of knowledge as code is a method in the language class. (That +of knowledge as code in a method in the language class. (That leaves a tidy distinction between the lexicon as the things we know how to I, and the rest of the things in the lexicon class @@ -658,5 +661,5 @@ Bracket Notation is a crucial feature of Locale::Maketext. I mean -Bracket Notation to provide a replacement for sprintf formatting. +Bracket Notation to provide a replacement for the use of sprintf formatting. Everything you do with Bracket Notation could be done with a sub block, but bracket notation is meant to be much more concise. @@ -664,7 +667,7 @@ Bracket Notation is a like a miniature "template" system (in the sense of L, not in the sense of C++ templates), -where normal text is passed thru basically as is, but text is special -regions is specially interpreted. In Bracket Notation, you use brackets -("[...]" -- not "{...}"!) to note sections that are specially interpreted. +where normal text is passed thru basically as is, but text in special +regions is specially interpreted. In Bracket Notation, you use square brackets ("[...]"), +not curly braces ("{...}") to note sections that are specially interpreted. For example, here all the areas that are taken literally are underlined with @@ -707,5 +710,5 @@ An item that is "_I" or "_-I" is interpreted as -$_[I]. I.e., "_1" is becomes with $_[1], and "_-3" is interpreted +$_[I]. I.e., "_1" becomes with $_[1], and "_-3" is interpreted as $_[-3] (in which case @_ should have at least three elements in it). Note that $_[0] is the language handle, and is typically not named @@ -747,5 +750,5 @@ =item * -If the first item in a bracket group is empty-string, or "_*" +If the first item in a bracket group is the empty-string, or "_*" or "_I" or "_-I", then that group is interpreted as just the interpolation of all its items: @@ -756,5 +759,5 @@ Examples: "[_1]" and "[,_1]", which are synonymous; and -"[,ID-(,_4,-,_2,)]", which compiles as +"C<[,ID-(,_4,-,_2,)]>", which compiles as C. @@ -762,5 +765,5 @@ Otherwise this bracket group is invalid. For example, in the group -"[!@#,whatever]", the first item C<"!@#"> is neither empty-string, +"[!@#,whatever]", the first item C<"!@#"> is neither the empty-string, "_I", "_-I", "_*", nor a valid method name; and so Locale::Maketext will throw an exception of you try compiling an @@ -791,5 +794,5 @@ return join '', "Hoohah ", - $lh->foo(" _1 ", " bar ", "baz"), #!!! + $lh->foo(" _1 ", " bar ", "baz"), # note the in " bar " "!", } @@ -812,5 +815,5 @@ Currently, an unescaped "~" before a character other than a bracket or a comma is taken to mean just a "~" and that -charecter. I.e., "~X" means the same as "~~X" -- i.e., one literal tilde, +character. I.e., "~X" means the same as "~~X" -- i.e., one literal tilde, and then one literal "X". However, by using "~X", you are assuming that no future version of Maketext will use "~X" as a magic escape sequence. @@ -869,5 +872,5 @@ I can picture all sorts of circumstances where you just do not want lookup to be able to fail (since failing -normally means that maketext throws a C, altho +normally means that maketext throws a C, although see the next section for greater control over that). But here's one circumstance where _AUTO lexicons are meant to @@ -880,5 +883,5 @@ go_process_file($filename) } else { - print "Couldn't find file \"$filename\"!\n"; + print qq{Couldn't find file "$filename"!\n}; } @@ -895,5 +898,5 @@ } else { print $lh->maketext( - "Couldn't find file \"[_1]\"!\n", $filename + qq{Couldn't find file "[_1]"!\n}, $filename ); } @@ -943,5 +946,5 @@ of its lexicons have C<_AUTO =E 1,>), then we have failed to find a normal way to maketext I. What then -happens in these failure conditions, depends on the $lh object +happens in these failure conditions, depends on the $lh object's "fail" attribute. @@ -954,5 +957,5 @@ coderef, then $lh->maketext(I,...params...) gives up and calls: - return &{$that_subref}($lh, $key, @params); + return $that_subref->($lh, $key, @params); Otherwise, the "fail" attribute's value should be a string denoting @@ -973,5 +976,5 @@ $lh->fail_with( undef ); - # Simply read: + # Get the current value $handler = $lh->fail_with(); @@ -985,5 +988,5 @@ # Make all lookups fall back onto an English value, - # but after we log it for later fingerpointing. + # but only after we log it for later fingerpointing. my $lh_backup = ThisProject->get_handle('en'); open(LEX_FAIL_LOG, ">>wherever/lex.log") || die "GNAARGH $!"; @@ -1005,8 +1008,8 @@ the "fail" attribute) to treat lookup failure as something other than an exception of the same level of severity as a config file being -unreadable, or some essential resource being inaccessable. +unreadable, or some essential resource being inaccessible. One possibly useful value for the "fail" attribute is the method name -"failure_handler_auto". This is a method defined in class +"failure_handler_auto". This is a method defined in the class Locale::Maketext itself. You set it with: @@ -1019,5 +1022,8 @@ But failure_handler_auto, instead of dying or anything, compiles -$key, caching it in $lh->{'failure_lex'}{$key} = $complied, +$key, caching it in + + $lh->{'failure_lex'}{$key} = $complied + and then calls the compiled value, and returns that. (I.e., if $key looks like bracket notation, $compiled is a sub, and we return @@ -1076,5 +1082,5 @@ my $lh = Projname::L10N->get_handle(...) || die "Language?"; -Assuming your call your class Projname::L10N, create a class +Assuming you call your class Projname::L10N, create a class consisting minimally of: @@ -1153,7 +1159,7 @@ =item * -You may at this point want to consider whether the your base class -(Projname::L10N) that all lexicons inherit from (Projname::L10N::en, -Projname::L10N::es, etc.) should be an _AUTO lexicon. It may be true +You may at this point want to consider whether your base class +(Projname::L10N), from which all lexicons inherit from (Projname::L10N::en, +Projname::L10N::es, etc.), should be an _AUTO lexicon. It may be true that in theory, all needed messages will be in each language class; but in the presumably unlikely or "impossible" case of lookup failure, @@ -1168,5 +1174,5 @@ (You may, in fact, want to start with localizing to I other language -at first, if you're not sure that you've property abstracted the +at first, if you're not sure that you've properly abstracted the language-dependent parts of your code.) @@ -1200,8 +1206,8 @@ appropriate. Typical variables in number formatting are: what to use as a decimal point (comma? period?); what to use as a thousands -separator (space? nonbreakinng space? comma? period? small +separator (space? nonbreaking space? comma? period? small middot? prime? apostrophe?); and even whether the so-called "thousands separator" is actually for every third digit -- I've heard reports of -two hundred thousand being expressable as "2,00,000" for some Indian +two hundred thousand being expressible as "2,00,000" for some Indian (Subcontinental) languages, besides the less surprising "S<200 000>", "200.000", "200,000", and "200'000". Also, using a set of numeral @@ -1272,9 +1278,10 @@ Journal> article about Maketext. It explains many important concepts underlying Locale::Maketext's design, and some insight into why -Maketext is better than the plain old approach of just having +Maketext is better than the plain old approach of having message catalogs that are just databases of sprintf formats. L is a sample application/module -that uses Locale::Maketext to localize its messages. +that uses Locale::Maketext to localize its messages. For a larger +internationalized system, see also L. L. @@ -1304,5 +1311,5 @@ =head1 COPYRIGHT AND DISCLAIMER -Copyright (c) 1999-2001 Sean M. Burke. All rights reserved. +Copyright (c) 1999-2004 Sean M. Burke. All rights reserved. This library is free software; you can redistribute it and/or modify @@ -1318,4 +1325,2 @@ =cut - -# Zing! Index: /branches/release-40/extlib/Locale/Maketext.pm =================================================================== --- /branches/release-40/extlib/Locale/Maketext.pm (revision 1098) +++ /branches/release-40/extlib/Locale/Maketext.pm (revision 2594) @@ -1,30 +1,27 @@ - -# Time-stamp: "2001-06-21 23:09:33 MDT" - -require 5; package Locale::Maketext; use strict; use vars qw( @ISA $VERSION $MATCH_SUPERS $USING_LANGUAGE_TAGS - $USE_LITERALS); +$USE_LITERALS $MATCH_SUPERS_TIGHTLY); use Carp (); -use I18N::LangTags 0.21 (); +use I18N::LangTags 0.30 (); #-------------------------------------------------------------------------- BEGIN { unless(defined &DEBUG) { *DEBUG = sub () {0} } } - # define the constant 'DEBUG' at compile-time - -$VERSION = "1.03"; +# define the constant 'DEBUG' at compile-time + +$VERSION = '1.13'; @ISA = (); $MATCH_SUPERS = 1; -$USING_LANGUAGE_TAGS = 1; - # Turning this off is somewhat of a security risk in that little or no - # checking will be done on the legality of tokens passed to the - # eval("use $module_name") in _try_use. If you turn this off, you have - # to do your own taint checking. +$MATCH_SUPERS_TIGHTLY = 1; +$USING_LANGUAGE_TAGS = 1; +# Turning this off is somewhat of a security risk in that little or no +# checking will be done on the legality of tokens passed to the +# eval("use $module_name") in _try_use. If you turn this off, you have +# to do your own taint checking. $USE_LITERALS = 1 unless defined $USE_LITERALS; - # a hint for compiling bracket-notation things. +# a hint for compiling bracket-notation things. my %isa_scan = (); @@ -33,27 +30,28 @@ sub quant { - my($handle, $num, @forms) = @_; - - return $num if @forms == 0; # what should this mean? - return $forms[2] if @forms > 2 and $num == 0; # special zeroth case - - # Normal case: - # Note that the formatting of $num is preserved. - return( $handle->numf($num) . ' ' . $handle->numerate($num, @forms) ); - # Most human languages put the number phrase before the qualified phrase. + my($handle, $num, @forms) = @_; + + return $num if @forms == 0; # what should this mean? + return $forms[2] if @forms > 2 and $num == 0; # special zeroth case + + # Normal case: + # Note that the formatting of $num is preserved. + return( $handle->numf($num) . ' ' . $handle->numerate($num, @forms) ); + # Most human languages put the number phrase before the qualified phrase. } sub numerate { - # return this lexical item in a form appropriate to this number - my($handle, $num, @forms) = @_; - my $s = ($num == 1); - - return '' unless @forms; - if(@forms == 1) { # only the headword form specified - return $s ? $forms[0] : ($forms[0] . 's'); # very cheap hack. - } else { # sing and plural were specified - return $s ? $forms[0] : $forms[1]; - } + # return this lexical item in a form appropriate to this number + my($handle, $num, @forms) = @_; + my $s = ($num == 1); + + return '' unless @forms; + if(@forms == 1) { # only the headword form specified + return $s ? $forms[0] : ($forms[0] . 's'); # very cheap hack. + } + else { # sing and plural were specified + return $s ? $forms[0] : $forms[1]; + } } @@ -61,26 +59,27 @@ sub numf { - my($handle, $num) = @_[0,1]; - if($num < 10_000_000_000 and $num > -10_000_000_000 and $num == int($num)) { - $num += 0; # Just use normal integer stringification. - # Specifically, don't let %G turn ten million into 1E+007 - } else { - $num = CORE::sprintf("%G", $num); - # "CORE::" is there to avoid confusion with the above sub sprintf. - } - while( $num =~ s/^([-+]?\d+)(\d{3})/$1,$2/s ) {1} # right from perlfaq5 - # The initial \d+ gobbles as many digits as it can, and then we - # backtrack so it un-eats the rightmost three, and then we - # insert the comma there. - - $num =~ tr<.,><,.> if ref($handle) and $handle->{'numf_comma'}; - # This is just a lame hack instead of using Number::Format - return $num; + my($handle, $num) = @_[0,1]; + if($num < 10_000_000_000 and $num > -10_000_000_000 and $num == int($num)) { + $num += 0; # Just use normal integer stringification. + # Specifically, don't let %G turn ten million into 1E+007 + } + else { + $num = CORE::sprintf('%G', $num); + # "CORE::" is there to avoid confusion with the above sub sprintf. + } + while( $num =~ s/^([-+]?\d+)(\d{3})/$1,$2/s ) {1} # right from perlfaq5 + # The initial \d+ gobbles as many digits as it can, and then we + # backtrack so it un-eats the rightmost three, and then we + # insert the comma there. + + $num =~ tr<.,><,.> if ref($handle) and $handle->{'numf_comma'}; + # This is just a lame hack instead of using Number::Format + return $num; } sub sprintf { - no integer; - my($handle, $format, @params) = @_; - return CORE::sprintf($format, @params); + no integer; + my($handle, $format, @params) = @_; + return CORE::sprintf($format, @params); # "CORE::" is there to avoid confusion with myself! } @@ -91,18 +90,18 @@ sub language_tag { - my $it = ref($_[0]) || $_[0]; - return undef unless $it =~ m/([^':]+)(?:::)?$/s; - $it = lc($1); - $it =~ tr<_><->; - return $it; + my $it = ref($_[0]) || $_[0]; + return undef unless $it =~ m/([^':]+)(?:::)?$/s; + $it = lc($1); + $it =~ tr<_><->; + return $it; } sub encoding { - my $it = $_[0]; - return( - (ref($it) && $it->{'encoding'}) - || "iso-8859-1" # Latin-1 - ); -} + my $it = $_[0]; + return( + (ref($it) && $it->{'encoding'}) + || 'iso-8859-1' # Latin-1 + ); +} #-------------------------------------------------------------------------- @@ -115,8 +114,8 @@ sub fail_with { # an actual attribute method! - my($handle, @params) = @_; - return unless ref($handle); - $handle->{'fail'} = $params[0] if @params; - return $handle->{'fail'}; + my($handle, @params) = @_; + return unless ref($handle); + $handle->{'fail'} = $params[0] if @params; + return $handle->{'fail'}; } @@ -124,33 +123,38 @@ sub failure_handler_auto { - # Meant to be used like: - # $handle->fail_with('failure_handler_auto') - - my($handle, $phrase, @params) = @_; - $handle->{'failure_lex'} ||= {}; - my $lex = $handle->{'failure_lex'}; - - my $value; - $lex->{$phrase} ||= ($value = $handle->_compile($phrase)); - - # Dumbly copied from sub maketext: - { - local $SIG{'__DIE__'}; - eval { $value = &$value($handle, @_) }; - } - # If we make it here, there was an exception thrown in the - # call to $value, and so scream: - if($@) { - my $err = $@; - # pretty up the error message - $err =~ s<\s+at\s+\(eval\s+\d+\)\s+line\s+(\d+)\.?\n?> - <\n in bracket code [compiled line $1],>s; - #$err =~ s/\n?$/\n/s; - Carp::croak "Error in maketexting \"$phrase\":\n$err as used"; - # Rather unexpected, but suppose that the sub tried calling - # a method that didn't exist. - } else { - return $value; - } + # Meant to be used like: + # $handle->fail_with('failure_handler_auto') + + my $handle = shift; + my $phrase = shift; + + $handle->{'failure_lex'} ||= {}; + my $lex = $handle->{'failure_lex'}; + + my $value; + $lex->{$phrase} ||= ($value = $handle->_compile($phrase)); + + # Dumbly copied from sub maketext: + return ${$value} if ref($value) eq 'SCALAR'; + return $value if ref($value) ne 'CODE'; + { + local $SIG{'__DIE__'}; + eval { $value = &$value($handle, @_) }; + } + # If we make it here, there was an exception thrown in the + # call to $value, and so scream: + if($@) { + my $err = $@; + # pretty up the error message + $err =~ s{\s+at\s+\(eval\s+\d+\)\s+line\s+(\d+)\.?\n?} + {\n in bracket code [compiled line $1],}s; + #$err =~ s/\n?$/\n/s; + Carp::croak "Error in maketexting \"$phrase\":\n$err as used"; + # Rather unexpected, but suppose that the sub tried calling + # a method that didn't exist. + } + else { + return $value; + } } @@ -158,9 +162,9 @@ sub new { - # Nothing fancy! - my $class = ref($_[0]) || $_[0]; - my $handle = bless {}, $class; - $handle->init; - return $handle; + # Nothing fancy! + my $class = ref($_[0]) || $_[0]; + my $handle = bless {}, $class; + $handle->init; + return $handle; } @@ -170,74 +174,82 @@ sub maketext { - # Remember, this can fail. Failure is controllable many ways. - Carp::croak "maketext requires at least one parameter" unless @_ > 1; - - my($handle, $phrase) = splice(@_,0,2); - - # Look up the value: - - my $value; - foreach my $h_r ( - @{ $isa_scan{ref($handle) || $handle} || $handle->_lex_refs } - ) { - print "* Looking up \"$phrase\" in $h_r\n" if DEBUG; - if(exists $h_r->{$phrase}) { - print " Found \"$phrase\" in $h_r\n" if DEBUG; - unless(ref($value = $h_r->{$phrase})) { - # Nonref means it's not yet compiled. Compile and replace. - $value = $h_r->{$phrase} = $handle->_compile($value); - } - last; - } elsif($phrase !~ m/^_/s and $h_r->{'_AUTO'}) { - # it's an auto lex, and this is an autoable key! - print " Automaking \"$phrase\" into $h_r\n" if DEBUG; - - $value = $h_r->{$phrase} = $handle->_compile($phrase); - last; - } - print " Not found in $h_r, nor automakable\n" if DEBUG > 1; - # else keep looking - } - - unless(defined($value)) { - print "! Lookup of \"$phrase\" in/under ", ref($handle) || $handle, - " fails.\n" if DEBUG; - if(ref($handle) and $handle->{'fail'}) { - print "WARNING0: maketext fails looking for <$phrase>\n" if DEBUG; - my $fail; - if(ref($fail = $handle->{'fail'}) eq 'CODE') { # it's a sub reference - return &{$fail}($handle, $phrase, @_); - # If it ever returns, it should return a good value. - } else { # It's a method name - return $handle->$fail($phrase, @_); - # If it ever returns, it should return a good value. - } - } else { - # All we know how to do is this; - Carp::croak("maketext doesn't know how to say:\n$phrase\nas needed"); - } - } - - return $$value if ref($value) eq 'SCALAR'; - return $value unless ref($value) eq 'CODE'; - - { - local $SIG{'__DIE__'}; - eval { $value = &$value($handle, @_) }; - } - # If we make it here, there was an exception thrown in the - # call to $value, and so scream: - if($@) { - my $err = $@; - # pretty up the error message - $err =~ s<\s+at\s+\(eval\s+\d+\)\s+line\s+(\d+)\.?\n?> - <\n in bracket code [compiled line $1],>s; - #$err =~ s/\n?$/\n/s; - Carp::croak "Error in maketexting \"$phrase\":\n$err as used"; - # Rather unexpected, but suppose that the sub tried calling - # a method that didn't exist. - } else { - return $value; - } + # Remember, this can fail. Failure is controllable many ways. + Carp::croak 'maketext requires at least one parameter' unless @_ > 1; + + my($handle, $phrase) = splice(@_,0,2); + Carp::confess('No handle/phrase') unless (defined($handle) && defined($phrase)); + + + # Don't interefere with $@ in case that's being interpolated into the msg. + local $@; + + # Look up the value: + + my $value; + foreach my $h_r ( + @{ $isa_scan{ref($handle) || $handle} || $handle->_lex_refs } + ) { + DEBUG and warn "* Looking up \"$phrase\" in $h_r\n"; + if(exists $h_r->{$phrase}) { + DEBUG and warn " Found \"$phrase\" in $h_r\n"; + unless(ref($value = $h_r->{$phrase})) { + # Nonref means it's not yet compiled. Compile and replace. + $value = $h_r->{$phrase} = $handle->_compile($value); + } + last; + } + elsif($phrase !~ m/^_/s and $h_r->{'_AUTO'}) { + # it's an auto lex, and this is an autoable key! + DEBUG and warn " Automaking \"$phrase\" into $h_r\n"; + + $value = $h_r->{$phrase} = $handle->_compile($phrase); + last; + } + DEBUG>1 and print " Not found in $h_r, nor automakable\n"; + # else keep looking + } + + unless(defined($value)) { + DEBUG and warn "! Lookup of \"$phrase\" in/under ", ref($handle) || $handle, " fails.\n"; + if(ref($handle) and $handle->{'fail'}) { + DEBUG and warn "WARNING0: maketext fails looking for <$phrase>\n"; + my $fail; + if(ref($fail = $handle->{'fail'}) eq 'CODE') { # it's a sub reference + return &{$fail}($handle, $phrase, @_); + # If it ever returns, it should return a good value. + } + else { # It's a method name + return $handle->$fail($phrase, @_); + # If it ever returns, it should return a good value. + } + } + else { + # All we know how to do is this; + Carp::croak("maketext doesn't know how to say:\n$phrase\nas needed"); + } + } + + return $$value if ref($value) eq 'SCALAR'; + return $value unless ref($value) eq 'CODE'; + + { + local $SIG{'__DIE__'}; + eval { $value = &$value($handle, @_) }; + } + # If we make it here, there was an exception thrown in the + # call to $value, and so scream: + if ($@) { + my $err = $@; + # pretty up the error message + $err =~ s{\s+at\s+\(eval\s+\d+\)\s+line\s+(\d+)\.?\n?} + {\n in bracket code [compiled line $1],}s; + #$err =~ s/\n?$/\n/s; + Carp::croak "Error in maketexting \"$phrase\":\n$err as used"; + # Rather unexpected, but suppose that the sub tried calling + # a method that didn't exist. + } + else { + return $value; + } } @@ -245,83 +257,127 @@ sub get_handle { # This is a constructor and, yes, it CAN FAIL. - # Its class argument has to be the base class for the current - # application's l10n files. - my($base_class, @languages) = @_; - $base_class = ref($base_class) || $base_class; - # Complain if they use __PACKAGE__ as a project base class? - - unless(@languages) { # Calling with no args is magical! wooo, magic! - if(length( $ENV{'REQUEST_METHOD'} || '' )) { # I'm a CGI - my $in = $ENV{'HTTP_ACCEPT_LANGUAGE'} || ''; - # supposedly that works under mod_perl, too. - $in =~ s<\([\)]*\)><>g; # Kill parens'd things -- just a hack. - @languages = &I18N::LangTags::extract_language_tags($in) if length $in; - # ...which untaints, incidentally. - - } else { # Not running as a CGI: try to puzzle out from the environment - if(length( $ENV{'LANG'} || '' )) { - push @languages, split m/[,:]/, $ENV{'LANG'}; - # LANG can be only /one/ locale as far as I know, but what the hey. - } - if(length( $ENV{'LANGUAGE'} || '' )) { - push @languages, split m/[,:]/, $ENV{'LANGUAGE'}; - } - print "Noting ENV LANG ", join(',', @languages),"\n" if DEBUG; - # Those are really locale IDs, but they get xlated a few lines down. - - if(&_try_use('Win32::Locale')) { - # If we have that module installed... - push @languages, Win32::Locale::get_language() - if defined &Win32::Locale::get_language; - } - } - } - - #------------------------------------------------------------------------ - print "Lgs1: ", map("<$_>", @languages), "\n" if DEBUG; - - if($USING_LANGUAGE_TAGS) { - @languages = map &I18N::LangTags::locale2language_tag($_), @languages; - # if it's a lg tag, fine, pass thru (untainted) - # if it's a locale ID, try converting to a lg tag (untainted), - # otherwise nix it. - - push @languages, map I18N::LangTags::super_languages($_), @languages - if $MATCH_SUPERS; - - @languages = map { $_, I18N::LangTags::alternate_language_tags($_) } - @languages; # catch alternation - - push @languages, I18N::LangTags::panic_languages(@languages) - if defined &I18N::LangTags::panic_languages; - - push @languages, $base_class->fallback_languages; - # You are free to override fallback_languages to return empty-list! - - @languages = # final bit of processing: - map { - my $it = $_; # copy - $it =~ tr<-A-Z><_a-z>; # lc, and turn - to _ - $it =~ tr<_a-z0-9><>cd; # remove all but a-z0-9_ - $it; - } @languages - ; - } - print "Lgs2: ", map("<$_>", @languages), "\n" if DEBUG > 1; - - push @languages, $base_class->fallback_language_classes; - # You are free to override that to return whatever. - - - my %seen = (); - foreach my $module_name ( map { $base_class . "::" . $_ } @languages ) - { - next unless length $module_name; # sanity - next if $seen{$module_name}++ # Already been here, and it was no-go - || !&_try_use($module_name); # Try to use() it, but can't it. - return($module_name->new); # Make it! - } - - return undef; # Fail! + # Its class argument has to be the base class for the current + # application's l10n files. + + my($base_class, @languages) = @_; + $base_class = ref($base_class) || $base_class; + # Complain if they use __PACKAGE__ as a project base class? + + if( @languages ) { + DEBUG and warn 'Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + if($USING_LANGUAGE_TAGS) { # An explicit language-list was given! + @languages = + map {; $_, I18N::LangTags::alternate_language_tags($_) } + # Catch alternation + map I18N::LangTags::locale2language_tag($_), + # If it's a lg tag, fine, pass thru (untainted) + # If it's a locale ID, try converting to a lg tag (untainted), + # otherwise nix it. + @languages; + DEBUG and warn 'Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + } + } + else { + @languages = $base_class->_ambient_langprefs; + } + + @languages = $base_class->_langtag_munging(@languages); + + my %seen; + foreach my $module_name ( map { $base_class . '::' . $_ } @languages ) { + next unless length $module_name; # sanity + next if $seen{$module_name}++ # Already been here, and it was no-go + || !&_try_use($module_name); # Try to use() it, but can't it. + return($module_name->new); # Make it! + } + + return undef; # Fail! +} + +########################################################################### + +sub _langtag_munging { + my($base_class, @languages) = @_; + + # We have all these DEBUG statements because otherwise it's hard as hell + # to diagnose ifwhen something goes wrong. + + DEBUG and warn 'Lgs1: ', map("<$_>", @languages), "\n"; + + if($USING_LANGUAGE_TAGS) { + DEBUG and warn 'Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + @languages = $base_class->_add_supers( @languages ); + + push @languages, I18N::LangTags::panic_languages(@languages); + DEBUG and warn "After adding panic languages:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + + push @languages, $base_class->fallback_languages; + # You are free to override fallback_languages to return empty-list! + DEBUG and warn 'Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + + @languages = # final bit of processing to turn them into classname things + map { + my $it = $_; # copy + $it =~ tr<-A-Z><_a-z>; # lc, and turn - to _ + $it =~ tr<_a-z0-9><>cd; # remove all but a-z0-9_ + $it; + } @languages + ; + DEBUG and warn "Nearing end of munging:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + } + else { + DEBUG and warn "Bypassing language-tags.\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + } + + DEBUG and warn "Before adding fallback classes:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + + push @languages, $base_class->fallback_language_classes; + # You are free to override that to return whatever. + + DEBUG and warn "Finally:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + + return @languages; +} + +########################################################################### + +sub _ambient_langprefs { + require I18N::LangTags::Detect; + return I18N::LangTags::Detect::detect(); +} + +########################################################################### + +sub _add_supers { + my($base_class, @languages) = @_; + + if (!$MATCH_SUPERS) { + # Nothing + DEBUG and warn "Bypassing any super-matching.\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + + } + elsif( $MATCH_SUPERS_TIGHTLY ) { + DEBUG and warn "Before adding new supers tightly:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + @languages = I18N::LangTags::implicate_supers( @languages ); + DEBUG and warn "After adding new supers tightly:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + + } + else { + DEBUG and warn "Before adding supers to end:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + @languages = I18N::LangTags::implicate_supers_strictly( @languages ); + DEBUG and warn "After adding supers to end:\n", + ' Lgs@', __LINE__, ': ', map("<$_>", @languages), "\n"; + } + + return @languages; } @@ -332,309 +388,35 @@ ########################################################################### -sub _compile { - # This big scarp routine compiles an entry. - # It returns either a coderef if there's brackety bits in this, or - # otherwise a ref to a scalar. - - my $target = ref($_[0]) || $_[0]; - - my(@code); - my(@c) = (''); # "chunks" -- scratch. - my $call_count = 0; - my $big_pile = ''; - { - my $in_group = 0; # start out outside a group - my($m, @params); # scratch - - while($_[1] =~ # Iterate over chunks. - m<\G( - [^\~\[\]]+ # non-~[] stuff - | - ~. # ~[, ~], ~~, ~other - | - \[ # [ presumably opening a group - | - \] # ] presumably closing a group - | - ~ # terminal ~ ? - | - $ - )>xgs - ) { - print " \"$1\"\n" if DEBUG > 2; - - if($1 eq '[' or $1 eq '') { # "[" or end - # Whether this is "[" or end, force processing of any - # preceding literal. - if($in_group) { - if($1 eq '') { - $target->_die_pointing($_[1], "Unterminated bracket group"); - } else { - $target->_die_pointing($_[1], "You can't nest bracket groups"); - } - } else { - if($1 eq '') { - print " [end-string]\n" if DEBUG > 2; - } else { - $in_group = 1; - } - die "How come \@c is empty?? in <$_[1]>" unless @c; # sanity - if(length $c[-1]) { - # Now actually processing the preceding literal - $big_pile .= $c[-1]; - if($USE_LITERALS and ( - (ord('A') == 65) - ? $c[-1] !~ m<[^\x20-\x7E]>s - # ASCII very safe chars - : $c[-1] !~ m/[^ !"\#\$%&'()*+,\-.\/0-9:;<=>?\@A-Z[\\\]^_`a-z{|}~\x07]/s - # EBCDIC very safe chars - )) { - # normal case -- all very safe chars - $c[-1] =~ s/'/\\'/g; - push @code, q{ '} . $c[-1] . "',\n"; - $c[-1] = ''; # reuse this slot - } else { - push @code, ' $c[' . $#c . "],\n"; - push @c, ''; # new chunk - } - } - # else just ignore the empty string. - } - - } elsif($1 eq ']') { # "]" - # close group -- go back in-band - if($in_group) { - $in_group = 0; - - print " --Closing group [$c[-1]]\n" if DEBUG > 2; - - # And now process the group... - - if(!length($c[-1]) or $c[-1] =~ m/^\s+$/s) { - DEBUG > 2 and print " -- (Ignoring)\n"; - $c[-1] = ''; # reset out chink - next; - } - - #$c[-1] =~ s/^\s+//s; - #$c[-1] =~ s/\s+$//s; - ($m,@params) = split(",", $c[-1], -1); # was /\s*,\s*/ - - # A bit of a hack -- we've turned "~,"'s into DELs, so turn - # 'em into real commas here. - if (ord('A') == 65) { # ASCII, etc - foreach($m, @params) { tr/\x7F/,/ } - } else { # EBCDIC (1047, 0037, POSIX-BC) - # Thanks to Peter Prymmer for the EBCDIC handling - foreach($m, @params) { tr/\x07/,/ } - } - - # Special-case handling of some method names: - if($m eq '_*' or $m =~ m<^_(-?\d+)$>s) { - # Treat [_1,...] as [,_1,...], etc. - unshift @params, $m; - $m = ''; - } elsif($m eq '*') { - $m = 'quant'; # "*" for "times": "4 cars" is 4 times "cars" - } elsif($m eq '#') { - $m = 'numf'; # "#" for "number": [#,_1] for "the number _1" - } - - # Most common case: a simple, legal-looking method name - if($m eq '') { - # 0-length method name means to just interpolate: - push @code, ' ('; - } elsif($m =~ m<^\w+(?:\:\:\w+)*$>s - and $m !~ m<(?:^|\:)\d>s - # exclude starting a (sub)package or symbol with a digit - ) { - # Yes, it even supports the demented (and undocumented?) - # $obj->Foo::bar(...) syntax. - $target->_die_pointing( - $_[1], "Can't (yet?) use \"SUPER::\" in a bracket-group method", - 2 + length($c[-1]) - ) - if $m =~ m/^SUPER::/s; - # Because for SUPER:: to work, we'd have to compile this into - # the right package, and that seems just not worth the bother, - # unless someone convinces me otherwise. - - push @code, ' $_[0]->' . $m . '('; - } else { - # TODO: implement something? or just too icky to consider? - $target->_die_pointing( - $_[1], - "Can't use \"$m\" as a method name in bracket group", - 2 + length($c[-1]) - ); - } - - pop @c; # we don't need that chunk anymore - ++$call_count; - - foreach my $p (@params) { - if($p eq '_*') { - # Meaning: all parameters except $_[0] - $code[-1] .= ' @_[1 .. $#_], '; - # and yes, that does the right thing for all @_ < 3 - } elsif($p =~ m<^_(-?\d+)$>s) { - # _3 meaning $_[3] - $code[-1] .= '$_[' . (0 + $1) . '], '; - } elsif($USE_LITERALS and ( - (ord('A') == 65) - ? $p !~ m<[^\x20-\x7E]>s - # ASCII very safe chars - : $p !~ m/[^ !"\#\$%&'()*+,\-.\/0-9:;<=>?\@A-Z[\\\]^_`a-z{|}~\x07]/s - # EBCDIC very safe chars - )) { - # Normal case: a literal containing only safe characters - $p =~ s/'/\\'/g; - $code[-1] .= q{'} . $p . q{', }; - } else { - # Stow it on the chunk-stack, and just refer to that. - push @c, $p; - push @code, ' $c[' . $#c . "], "; - } - } - $code[-1] .= "),\n"; - - push @c, ''; - } else { - $target->_die_pointing($_[1], "Unbalanced ']'"); - } - - } elsif(substr($1,0,1) ne '~') { - # it's stuff not containing "~" or "[" or "]" - # i.e., a literal blob - $c[-1] .= $1; - - } elsif($1 eq '~~') { # "~~" - $c[-1] .= '~'; - - } elsif($1 eq '~[') { # "~[" - $c[-1] .= '['; - - } elsif($1 eq '~]') { # "~]" - $c[-1] .= ']'; - - } elsif($1 eq '~,') { # "~," - if($in_group) { - # This is a hack, based on the assumption that no-one will actually - # want a DEL inside a bracket group. Let's hope that's it's true. - if (ord('A') == 65) { # ASCII etc - $c[-1] .= "\x7F"; - } else { # EBCDIC (cp 1047, 0037, POSIX-BC) - $c[-1] .= "\x07"; - } - } else { - $c[-1] .= '~,'; - } - - } elsif($1 eq '~') { # possible only at string-end, it seems. - $c[-1] .= '~'; - - } else { - # It's a "~X" where X is not a special character. - # Consider it a literal ~ and X. - $c[-1] .= $1; - } - } - } - - if($call_count) { - undef $big_pile; # Well, nevermind that. - } else { - # It's all literals! Ahwell, that can happen. - # So don't bother with the eval. Return a SCALAR reference. - return \$big_pile; - } - - die "Last chunk isn't null??" if @c and length $c[-1]; # sanity - print scalar(@c), " chunks under closure\n" if DEBUG; - if(@code == 0) { # not possible? - print "Empty code\n" if DEBUG; - return \''; - } elsif(@code > 1) { # most cases, presumably! - unshift @code, "join '',\n"; - } - unshift @code, "use strict; sub {\n"; - push @code, "}\n"; - - print @code if DEBUG; - my $sub = eval(join '', @code); - die "$@ while evalling" . join('', @code) if $@; # Should be impossible. - return $sub; -} - -# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -sub _die_pointing { - # This is used by _compile to throw a fatal error - my $target = shift; # class name - # ...leaving $_[0] the error-causing text, and $_[1] the error message - - my $i = index($_[0], "\n"); - - my $pointy; - my $pos = pos($_[0]) - (defined($_[2]) ? $_[2] : 0) - 1; - if($pos < 1) { - $pointy = "^=== near there\n"; - } else { # we need to space over - my $first_tab = index($_[0], "\t"); - if($pos > 2 and ( -1 == $first_tab or $first_tab > pos($_[0]))) { - # No tabs, or the first tab is harmlessly after where we will point to, - # AND we're far enough from the margin that we can draw a proper arrow. - $pointy = ('=' x $pos) . "^ near there\n"; - } else { - # tabs screw everything up! - $pointy = substr($_[0],0,$pos); - $pointy =~ tr/\t //cd; - # make everything into whitespace, but preseving tabs - $pointy .= "^=== near there\n"; - } - } - - my $errmsg = "$_[1], in\:\n$_[0]"; - - if($i == -1) { - # No newline. - $errmsg .= "\n" . $pointy; - } elsif($i == (length($_[0]) - 1) ) { - # Already has a newline at end. - $errmsg .= $pointy; - } else { - # don't bother with the pointy bit, I guess. - } - Carp::croak( "$errmsg via $target, as used" ); -} +use Locale::Maketext::GutsLoader; ########################################################################### my %tried = (); - # memoization of whether we've used this module, or found it unusable. +# memoization of whether we've used this module, or found it unusable. sub _try_use { # Basically a wrapper around "require Modulename" - # "Many men have tried..." "They tried and failed?" "They tried and died." - return $tried{$_[0]} if exists $tried{$_[0]}; # memoization - - my $module = $_[0]; # ASSUME sane module name! - { no strict 'refs'; - return($tried{$module} = 1) - if defined(%{$module . "::Lexicon"}) or defined(@{$module . "::ISA"}); - # weird case: we never use'd it, but there it is! - } - - print " About to use $module ...\n" if DEBUG; - { - local $SIG{'__DIE__'}; - eval "require $module"; # used to be "use $module", but no point in that. - } - if($@) { - print "Error using $module \: $@\n" if DEBUG > 1; - return $tried{$module} = 0; - } else { - print " OK, $module is used\n" if DEBUG; - return $tried{$module} = 1; - } + # "Many men have tried..." "They tried and failed?" "They tried and died." + return $tried{$_[0]} if exists $tried{$_[0]}; # memoization + + my $module = $_[0]; # ASSUME sane module name! + { no strict 'refs'; + return($tried{$module} = 1) + if defined(%{$module . '::Lexicon'}) or defined(@{$module . '::ISA'}); + # weird case: we never use'd it, but there it is! + } + + DEBUG and warn " About to use $module ...\n"; + { + local $SIG{'__DIE__'}; + eval "require $module"; # used to be "use $module", but no point in that. + } + if($@) { + DEBUG and warn "Error using $module \: $@\n"; + return $tried{$module} = 0; + } + else { + DEBUG and warn " OK, $module is used\n"; + return $tried{$module} = 1; + } } @@ -642,34 +424,33 @@ sub _lex_refs { # report the lexicon references for this handle's class - # returns an arrayREF! - no strict 'refs'; - my $class = ref($_[0]) || $_[0]; - print "Lex refs lookup on $class\n" if DEBUG > 1; - return $isa_scan{$class} if exists $isa_scan{$class}; # memoization! - - my @lex_refs; - my $seen_r = ref($_[1]) ? $_[1] : {}; - - if( defined( *{$class . '::Lexicon'}{'HASH'} )) { - push @lex_refs, *{$class . '::Lexicon'}{'HASH'}; - print "%" . $class . "::Lexicon contains ", - scalar(keys %{$class . '::Lexicon'}), " entries\n" if DEBUG; - } - - # Implements depth(height?)-first recursive searching of superclasses. - # In hindsight, I suppose I could have just used Class::ISA! - foreach my $superclass (@{$class . "::ISA"}) { - print " Super-class search into $superclass\n" if DEBUG; - next if $seen_r->{$superclass}++; - push @lex_refs, @{&_lex_refs($superclass, $seen_r)}; # call myself - } - - $isa_scan{$class} = \@lex_refs; # save for next time - return \@lex_refs; + # returns an arrayREF! + no strict 'refs'; + no warnings 'once'; + my $class = ref($_[0]) || $_[0]; + DEBUG and warn "Lex refs lookup on $class\n"; + return $isa_scan{$class} if exists $isa_scan{$class}; # memoization! + + my @lex_refs; + my $seen_r = ref($_[1]) ? $_[1] : {}; + + if( defined( *{$class . '::Lexicon'}{'HASH'} )) { + push @lex_refs, *{$class . '::Lexicon'}{'HASH'}; + DEBUG and warn '%' . $class . '::Lexicon contains ', + scalar(keys %{$class . '::Lexicon'}), " entries\n"; + } + + # Implements depth(height?)-first recursive searching of superclasses. + # In hindsight, I suppose I could have just used Class::ISA! + foreach my $superclass (@{$class . '::ISA'}) { + DEBUG and warn " Super-class search into $superclass\n"; + next if $seen_r->{$superclass}++; + push @lex_refs, @{&_lex_refs($superclass, $seen_r)}; # call myself + } + + $isa_scan{$class} = \@lex_refs; # save for next time + return \@lex_refs; } sub clear_isa_scan { %isa_scan = (); return; } # end on a note of simplicity! -########################################################################### 1; -