论坛 › Erlang › 【mnesia】分片

shijiang1130 发表于 2015-02-03 23:01

【mnesia】分片

在其它数据库里也有分片的概念，原因是有大表，我这里有一个例子，数据是全球ip地址信息到城市，数据大约是460万，全load到内存我的笔记本6G,撑不住。用了分片（10）后，内存也占用不少，但是不至于load不起来。

shijiang1130 发表于 2015-02-03 23:01

-module(location).
-include_lib("include/location.hrl").
-include_lib("stdlib/include/qlc.hrl").
-compile(export_all).

read_txt() ->
    {ok, M} = file:open("dbip-city.csv", ),
    read_line(M).

read_line(M) ->
    case file:read_line(M) of
      {ok, L} -> i(L), read_line(M);
      {eof} -> ok, file:close(M)
    end.

i(S) ->
    Slash = binary:replace(S, <<"\"">>, <<>>, ),
    T = erlang:binary_to_list(Slash),
    R = string:tokens(T, "\n"),
    spawn(fun () -> insert_data(R) end).

insert_data(L) ->
    case string:tokens(lists:flatten(L), ",") of
       ->
          Row = #location{ip = Ip, mask = Mask, country = Country,
                          area = Area0, city = lists:concat()};
       ->
          Row = #location{ip = Ip, mask = Mask, country = Country,
                          area = Area, city = City};
       ->
          Row = #location{ip = Ip, mask = Mask, country = Country,
                          area = Area};
       ->
          Row = #location{ip = Ip, mask = Mask, country = Country}
    end,
    db_dirty_write(Row).

init_table() ->
    mnesia:start(),
    mnesia:delete_table(location),
    mnesia:create_table(location,
                        [{attributes, record_info(fields, location)},
                       {disc_only_copies, }]),
    mnesia:change_config(extra_db_nodes, ),
    mnesia:add_table_copy(location, nodes(),
                          disc_only_copies),
    read_txt().

address(Ip) ->
    MatchHead = #location{ip = '$4', mask = '$5',
                          country = '$1', area = '$2', city = '$3'},
    Guard = [{'>=', '$4', Ip}, {'=<', '$5', Ip}],
    Result = ['$4', '$5', '$1', '$2', '$3'],
    MatchSpec = [{MatchHead, Guard, Result}],
    SelFun = fun () ->
                     mnesia:select(location, MatchSpec, 10, read)
             end,
    mnesia:activity(transaction, SelFun, [], mnesia_frag).

query(Ip) ->
    Query = fun () ->
                  Q = qlc:q([E
                             || E <- mnesia:table(location),
                                  E#location.ip =< Ip]),
                  Sort = qlc:e(qlc:keysort(2, Q, [{order, descending}])),
                  QC = qlc:cursor(Sort),
                  qlc:next_answers(QC, 1)
          end,
    {atomic, Results} = mnesia:transaction(Query),
    Results.

q(Ip) ->
    Query = fun () ->
                  qlc:eval(qlc:q([[E#location.ip, E#location.mask,
                                     E#location.country, E#location.area,
                                     E#location.city]
                                  || E <- mnesia:table(location),
                                     (Ip =< E#location.mask) and
                                       (E#location.ip =< Ip)]))
          end,
    {atomic, Results} = mnesia:transaction(Query),
    Results.

q1() -> mnesia:dirty_read({location, "195.101.114.0"}).

q2() ->
    mnesia:match_object({location, "195.101.114.0"}).

db() ->
    mnesia:create_table(location,
                        [{frag_properties,
                          [{n_fragments, 20}, {n_disc_copies, 1},
                           {node_pool, }]},
                       %{index, },
                       {attributes, record_info(fields, location)}]).

db_dirty_write(Row) ->
    AddFun = fun () -> mnesia:write(location, Row, write)
             end,
    ok = mnesia:activity(async_dirty, AddFun, [],
                       mnesia_frag).

shijiang1130 发表于 2015-02-03 23:02

用这种数据学习mnesia我觉得是个好的方法。

shijiang1130 发表于 2015-02-03 23:03

你可以下载到数据 https://db-ip.com/db/#php 第二列IP address to city (low resolution)

查看完整版本: 【mnesia】分片