Sphinx Problem

Real Hal9000 · January 17, 2013

Hello to all

We're using sphinx 2.0.4 and we have some problems with the sphinx indexer

It takes too long to index to be usable, so i believe I am doing something wrong. See the attach of the sphinx.conf [1] that I use and the output of indexer --all [2] and indexer with only the deltas [3]

Any ideas ?

 [1]
root@web:~# cat /home/hal9000/sphinx/sphinx.conf

#############################################################################
## data source definition
#############################################################################

source ipb_source_config
{
#setup 
    type                    = mysql
    sql_host                = localhost
    sql_user                = hal9000
    sql_pass                = 2323a3
    sql_db                    = insomniacs
    sql_port                = 3306
}


############################### --- TAGS ---
################################

source core_tags_search_main : ipb_source_config
{
    # Set our forum PID counter
    sql_query_pre = SET NAMES utf8
    sql_query_pre    = REPLACE INTO cache_store VALUES(
'sphinx_core_tags_counter', (SELECT max(tag_id) FROM core_tags), 0,
UNIX_TIMESTAMP(), 0 )

    # Query posts for the main source
    sql_query        = SELECT t.tag_id, t.tag_id as search_id,
t.tag_added, t.tag_member_id, t.tag_meta_id, t.tag_meta_parent_id,
t.tag_text, t.tag_meta_app, REPLACE( t.tag_meta_area, '-', '_' ) as
tag_meta_area,  
                             CONCAT( ',', p.tag_perm_text, ',') as
tag_perm_text ,p.tag_perm_visible 
                      FROM core_tags t 
                      LEFT JOIN core_tags_perms p ON (
t.tag_aai_lookup=p.tag_perm_aai_lookup )

    # Fields
    sql_attr_uint       = search_id
    sql_attr_timestamp = tag_added
    sql_attr_uint      = tag_member_id
    sql_attr_uint      = tag_meta_id
    sql_attr_uint      = tag_meta_parent_id
    sql_attr_uint       = tag_perm_visible
    sql_ranged_throttle    = 0
}

source core_tags_search_delta : core_tags_search_main
{
    # Override the base sql_query_pre
    sql_query_pre = SET NAMES utf8
    sql_query_pre    =

    # Query posts for the main source
    sql_query        = SELECT t.tag_id, t.tag_id as search_id,
t.tag_added, t.tag_member_id, t.tag_meta_id, t.tag_meta_parent_id,
t.tag_text, t.tag_meta_app, REPLACE( t.tag_meta_area, '-', '_' ) as
tag_meta_area,  
                             CONCAT( ',', p.tag_perm_text, ',') as
tag_perm_text ,p.tag_perm_visible 
                      FROM core_tags t 
                      LEFT JOIN core_tags_perms p ON (
t.tag_aai_lookup=p.tag_perm_aai_lookup ) 
                      WHERE t.tag_id>  ( SELECT cs_value FROM
cache_store WHERE cs_key='sphinx_core_tags_counter' )
}

index core_tags_search_main
{
    source            = core_tags_search_main
    path            = /var/lib/sphinxsearch/data/core_tags_search_main

    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len    = 2
#    charset_type    = sbcs
    charset_type    = utf-8
    charset_table = U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z,
U+FF41..U+FF5A->a..z, 0..9, A..Z->a..z, a..z, U+0386->U+03B1,
U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF,
U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9,
U+03AB->U+03C5, U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7,
U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5,
U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2,
U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5,
U+03D5->U+03C6, U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB,
U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3,
U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB,
U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1,
U+03F2->U+03C3, U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5,
U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1,
U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1,
U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1,
U+1F09->U+03B1, U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1,
U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5,
U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5,
U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5,
U+1F1B->U+03B5, U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7,
U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7,
U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7,
U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7,
U+1F2D->U+03B7, U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9,
U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9,
U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9,
U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9,
U+1F3D->U+03B9, U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF,
U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF,
U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF,
U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5,
U+1F51->U+03C5, U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5,
U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5,
U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9,
U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9,
U+1F65->U+03C9, U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9,
U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9,
U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1,
U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7,
U+1F75->U+03B7, U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF,
U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9,
U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1,
U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1,
U+1F87->U+03B1, U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1,
U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1,
U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7,
U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7,
U+1F97->U+03B7, U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7,
U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7,
U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9,
U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9,
U+1FA7->U+03C9, U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9,
U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9,
U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1,
U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1,
U+1FB8->U+03B1, U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1,
U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7,
U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5,
U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9,
U+1FD1->U+03B9, U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9,
U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9,
U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5,
U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5,
U+1FE7->U+03C5, U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5,
U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9,
U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF,
U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9,
U+0391..U+03A1->U+03B1..U+03C1, U+03B1..U+03C1,
U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB,
U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED,
U+03EF, U+03F3, U+03F8, U+03FB
    html_strip        = 0
#infix_fields    = tag_text 
#min_infix_len   = 3 
#enable_star     = 1 
}

index core_tags_search_delta : core_tags_search_main
{
   source            = core_tags_search_delta
   path                = /var/lib/sphinxsearch/data/core_tags_search_delta
}

################################# --- FORUM ---
##############################
source forums_search_posts_main : ipb_source_config
{
    # Set our forum PID counter
    sql_query_pre = SET NAMES utf8
    sql_query_pre    = REPLACE INTO cache_store VALUES(
'sphinx_forums_counter_posts', (SELECT max(pid) FROM posts), 0,
UNIX_TIMESTAMP(), 0 )

    # Query posts for the main source
    sql_query        = SELECT p.pid, p.pid as search_id, p.author_id,
p.post_date, REPLACE( p.post, '-', '&#8208') as post, p.topic_id,
p.queued, 
                             t.tid, LOWER(t.title) as tordinal,
REPLACE( t.title, '-', '&#8208') as title, t.views, t.posts,
t.forum_id, t.last_post, t.state, t.start_date, t.starter_id,
t.last_poster_id, t.topic_firstpost, 
                            CASE WHEN t.approved = -1 THEN 1 ELSE 0 END
AS soft_deleted, 
                            CASE WHEN t.approved = -1 THEN 0 ELSE
t.approved END AS approved, 
                            CASE WHEN t.topic_archive_status IN (0,3)
THEN 0 ELSE 1 END AS archive_status, 
                            CONCAT( SUBSTRING( t.last_post, 2, 8 ),
'0', LPAD( t.tid, 10, 0 ) ) as last_post_group 
                      FROM posts p 
                      LEFT JOIN topics t ON ( p.topic_id=t.tid )

    # Fields
    sql_attr_uint            = queued
    sql_attr_uint            = approved
    sql_attr_uint            = soft_deleted
    sql_attr_uint            = archive_status
    sql_attr_uint            = search_id
    sql_attr_uint            = forum_id
    sql_attr_timestamp        = post_date
    sql_attr_timestamp        = last_post
    sql_attr_timestamp        = start_date
    sql_attr_uint            = author_id
    sql_attr_uint            = starter_id
    sql_attr_uint            = tid
    sql_attr_uint            = posts
    sql_attr_uint            = views
    sql_attr_str2ordinal    = tordinal
    sql_attr_bigint            = last_post_group
    sql_attr_multi            = uint tag_id from query; SELECT
t.topic_firstpost, c.tag_id FROM core_tags c LEFT JOIN topics t ON
(t.tid=c.tag_meta_id) WHERE c.tag_meta_app='forums' AND
c.tag_meta_area='topics'

    sql_ranged_throttle    = 0
}

source forums_search_posts_delta : forums_search_posts_main
{
    # Override the base sql_query_pre
    sql_query_pre = SET NAMES utf8
    sql_query_pre =

    # Query posts for the delta source
    sql_query        = SELECT p.pid, p.pid as search_id, p.author_id,
p.post_date, REPLACE( p.post, '-', '&#8208') as post, p.topic_id,
p.queued, 
                             t.tid, LOWER(t.title) as tordinal,
REPLACE( t.title, '-', '&#8208') as title, t.views, t.posts,
t.forum_id, t.last_post, t.state, t.start_date, t.starter_id,
t.last_poster_id, t.topic_firstpost, 
                             CASE WHEN t.approved = -1 THEN 1 ELSE 0
END AS soft_deleted, 
                              CASE WHEN t.approved = -1 THEN 0 ELSE
t.approved END AS approved, 
                              CASE WHEN t.topic_archive_status IN (0,3)
THEN 0 ELSE 1 END AS archive_status, 
                             CONCAT( SUBSTRING( t.last_post, 2, 8 ),
'0', LPAD( t.tid, 10, 0 ) ) as last_post_group 
                      FROM posts p 
                      LEFT JOIN topics t ON ( p.topic_id=t.tid ) 
                      WHERE p.pid>  ( SELECT cs_value FROM cache_store
WHERE cs_key='sphinx_forums_counter_posts' )

    sql_query_killlist = SELECT pid FROM posts WHERE pid>  ( SELECT
cs_value FROM cache_store WHERE cs_key='sphinx_forums_counter_posts' )
}

index forums_search_posts_main
{
    source            = forums_search_posts_main
    path            = /var/lib/sphinxsearch/data/forums_search_posts_main

    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len    = 2
#    charset_type    = sbcs
    charset_type    = utf-8
    charset_table = U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z,
U+FF41..U+FF5A->a..z, 0..9, A..Z->a..z, a..z, U+0386->U+03B1,
U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF,
U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9,
U+03AB->U+03C5, U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7,
U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5,
U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2,
U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5,
U+03D5->U+03C6, U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB,
U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3,
U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB,
U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1,
U+03F2->U+03C3, U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5,
U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1,
U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1,
U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1,
U+1F09->U+03B1, U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1,
U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5,
U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5,
U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5,
U+1F1B->U+03B5, U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7,
U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7,
U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7,
U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7,
U+1F2D->U+03B7, U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9,
U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9,
U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9,
U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9,
U+1F3D->U+03B9, U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF,
U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF,
U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF,
U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5,
U+1F51->U+03C5, U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5,
U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5,
U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9,
U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9,
U+1F65->U+03C9, U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9,
U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9,
U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1,
U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7,
U+1F75->U+03B7, U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF,
U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9,
U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1,
U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1,
U+1F87->U+03B1, U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1,
U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1,
U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7,
U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7,
U+1F97->U+03B7, U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7,
U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7,
U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9,
U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9,
U+1FA7->U+03C9, U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9,
U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9,
U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1,
U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1,
U+1FB8->U+03B1, U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1,
U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7,
U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5,
U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9,
U+1FD1->U+03B9, U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9,
U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9,
U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5,
U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5,
U+1FE7->U+03C5, U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5,
U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9,
U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF,
U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9,
U+0391..U+03A1->U+03B1..U+03C1, U+03B1..U+03C1,
U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB,
U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED,
U+03EF, U+03F3, U+03F8, U+03FB
    html_strip        = 0
#infix_fields    = post, title 
#min_infix_len   = 3 
#enable_star     = 1 
}

index forums_search_posts_delta : forums_search_posts_main
{
   source            = forums_search_posts_delta
   path                =
/var/lib/sphinxsearch/data/forums_search_posts_delta
}

source forums_search_archive_main : ipb_source_config
{
    # Set our forum PID counter
    sql_query_pre = SET NAMES utf8
    sql_query_pre    = REPLACE INTO cache_store VALUES(
'sphinx_forums_counter_archives', (SELECT max(archive_id) FROM
forums_archive_posts), 0, UNIX_TIMESTAMP(), 0 )

    # Query posts for the main source
    sql_query        = SELECT p.archive_id, p.archive_id as search_id,
p.archive_author_id as author_id, p.archive_content_date as post_date,
REPLACE( p.archive_content, '-', '&#8208') as archive_content,
p.archive_topic_id, p.archive_queued as queued, 
                             t.tid, LOWER(t.title) as tordinal,
REPLACE( t.title, '-', '&#8208') as title, t.views, t.posts,
t.forum_id, t.last_post, t.state, t.start_date, t.starter_id,
t.last_poster_id, t.topic_firstpost, 
                            CASE WHEN t.approved = -1 THEN 1 ELSE 0 END
AS soft_deleted, 
                            CASE WHEN t.approved = -1 THEN 0 ELSE
t.approved END AS approved, 
                            CASE WHEN t.topic_archive_status IN (0,3)
THEN 0 ELSE 1 END AS archive_status, 
                            CONCAT( SUBSTRING( t.last_post, 2, 8 ),
'0', LPAD( t.tid, 10, 0 ) ) as last_post_group 
                      FROM forums_archive_posts p 
                      LEFT JOIN topics t ON ( p.archive_topic_id=t.tid )

    # Fields
    sql_attr_uint            = queued
    sql_attr_uint            = approved
    sql_attr_uint            = soft_deleted
    sql_attr_uint            = archive_status
    sql_attr_uint            = search_id
    sql_attr_uint            = forum_id
    sql_attr_timestamp        = post_date
    sql_attr_timestamp        = last_post
    sql_attr_timestamp        = start_date
    sql_attr_uint            = author_id
    sql_attr_uint            = starter_id
    sql_attr_uint            = tid
    sql_attr_uint            = posts
    sql_attr_uint            = views
    sql_attr_str2ordinal    = tordinal
    sql_attr_bigint            = last_post_group
    sql_attr_multi            = uint tag_id from query; SELECT
t.topic_firstpost, c.tag_id FROM core_tags c LEFT JOIN topics t ON
(t.tid=c.tag_meta_id) WHERE c.tag_meta_app='forums' AND
c.tag_meta_area='topics'

    sql_ranged_throttle    = 0
}

source forums_search_archive_delta : forums_search_archive_main
{
    # Override the base sql_query_pre
    sql_query_pre = SET NAMES utf8
    sql_query_pre =

    # Query posts for the delta source
    sql_query        = SELECT p.archive_id, p.archive_id as search_id,
p.archive_author_id as author_id, p.archive_content_date as post_date,
REPLACE( p.archive_content, '-', '&#8208') as archive_content,
p.archive_topic_id, p.archive_queued as queued, 
                             t.tid, LOWER(t.title) as tordinal,
REPLACE( t.title, '-', '&#8208') as title, t.views, t.posts,
t.forum_id, t.last_post, t.state, t.start_date, t.starter_id,
t.last_poster_id, t.topic_firstpost, 
                             CASE WHEN t.approved = -1 THEN 1 ELSE 0
END AS soft_deleted, 
                              CASE WHEN t.approved = -1 THEN 0 ELSE
t.approved END AS approved, 
                             CASE WHEN t.topic_archive_status IN (0,3)
THEN 0 ELSE 1 END AS archive_status, 
                             CONCAT( SUBSTRING( t.last_post, 2, 8 ),
'0', LPAD( t.tid, 10, 0 ) ) as last_post_group 
                      FROM forums_archive_posts p 
                      LEFT JOIN topics t ON ( p.archive_topic_id=t.tid ) 
                      WHERE p.archive_id>  ( SELECT cs_value FROM
cache_store WHERE cs_key='sphinx_forums_counter_archives' )

    sql_query_killlist = SELECT archive_id FROM forums_archive_posts
WHERE archive_id>  ( SELECT cs_value FROM cache_store WHERE
cs_key='sphinx_forums_counter_archives' )
}

index forums_search_archive_main
{
    source            = forums_search_archive_main
    path            = /var/lib/sphinxsearch/data/forums_search_archive_main

    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len    = 2
#    charset_type    = sbcs
    charset_type    = utf-8
    charset_table = U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z,
U+FF41..U+FF5A->a..z, 0..9, A..Z->a..z, a..z, U+0386->U+03B1,
U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF,
U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9,
U+03AB->U+03C5, U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7,
U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5,
U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2,
U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5,
U+03D5->U+03C6, U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB,
U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3,
U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB,
U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1,
U+03F2->U+03C3, U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5,
U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1,
U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1,
U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1,
U+1F09->U+03B1, U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1,
U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5,
U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5,
U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5,
U+1F1B->U+03B5, U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7,
U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7,
U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7,
U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7,
U+1F2D->U+03B7, U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9,
U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9,
U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9,
U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9,
U+1F3D->U+03B9, U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF,
U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF,
U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF,
U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5,
U+1F51->U+03C5, U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5,
U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5,
U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9,
U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9,
U+1F65->U+03C9, U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9,
U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9,
U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1,
U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7,
U+1F75->U+03B7, U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF,
U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9,
U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1,
U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1,
U+1F87->U+03B1, U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1,
U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1,
U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7,
U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7,
U+1F97->U+03B7, U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7,
U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7,
U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9,
U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9,
U+1FA7->U+03C9, U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9,
U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9,
U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1,
U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1,
U+1FB8->U+03B1, U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1,
U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7,
U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5,
U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9,
U+1FD1->U+03B9, U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9,
U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9,
U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5,
U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5,
U+1FE7->U+03C5, U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5,
U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9,
U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF,
U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9,
U+0391..U+03A1->U+03B1..U+03C1, U+03B1..U+03C1,
U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB,
U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED,
U+03EF, U+03F3, U+03F8, U+03FB
    html_strip        = 0
#infix_fields    = archive_content, title 
#min_infix_len   = 3 
#enable_star     = 1 
}

index forums_search_archive_delta : forums_search_archive_main
{
   source            = forums_search_archive_delta
   path                =
/var/lib/sphinxsearch/data/forums_search_archive_delta
}

############################ --- DOWNLOADS ---
##############################

source downloads_search_main : ipb_source_config
{
    # Set our forum PID counter
    sql_query_pre    = REPLACE INTO cache_store VALUES(
'sphinx_downloads_counter', (SELECT max(file_id) FROM downloads_files),
0, UNIX_TIMESTAMP(), 0 )

    # Query posts for the main source
    sql_query        = SELECT f.*, f.file_id as search_id, f.file_name
as fordinal, cc.field_1 
                      FROM downloads_files f 
                      LEFT JOIN downloads_ccontent cc ON
(cc.file_id=f.file_id)

    # Fields
    sql_attr_uint            = search_id
    sql_attr_uint            = file_cat
    sql_attr_uint            = file_open
    sql_attr_uint            = file_views
    sql_attr_uint            = file_rating
    sql_attr_uint            = file_downloads
    sql_attr_timestamp        = file_updated
    sql_attr_timestamp        = file_submitted
    sql_attr_uint            = file_submitter
    sql_attr_str2ordinal    = fordinal
    sql_attr_float            = file_cost
    sql_attr_multi            = uint tag_id from query; SELECT
tag_meta_id, tag_id FROM core_tags WHERE tag_meta_app='downloads' AND
tag_meta_area='files'

    sql_ranged_throttle    = 0
}

source downloads_search_delta : downloads_search_main
{
    # Override the base sql_query_pre
    sql_query_pre    =

    # Query posts for the main source
    sql_query        = SELECT f.*, f.file_id as search_id, f.file_name
as fordinal, cc.field_1 
                      FROM downloads_files f 
                      LEFT JOIN downloads_ccontent cc ON
(cc.file_id=f.file_id) 
                      WHERE f.file_id>  ( SELECT cs_value FROM
cache_store WHERE cs_key='sphinx_downloads_counter' )
}

index downloads_search_main
{
    source            = downloads_search_main
    path            = /var/lib/sphinxsearch/data/downloads_search_main

    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len    = 2
#    charset_type    = sbcs
    charset_type    = utf-8
    charset_table = U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z,
U+FF41..U+FF5A->a..z, 0..9, A..Z->a..z, a..z, U+0386->U+03B1,
U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF,
U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9,
U+03AB->U+03C5, U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7,
U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5,
U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2,
U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5,
U+03D5->U+03C6, U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB,
U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3,
U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB,
U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1,
U+03F2->U+03C3, U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5,
U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1,
U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1,
U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1,
U+1F09->U+03B1, U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1,
U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5,
U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5,
U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5,
U+1F1B->U+03B5, U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7,
U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7,
U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7,
U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7,
U+1F2D->U+03B7, U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9,
U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9,
U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9,
U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9,
U+1F3D->U+03B9, U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF,
U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF,
U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF,
U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5,
U+1F51->U+03C5, U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5,
U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5,
U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9,
U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9,
U+1F65->U+03C9, U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9,
U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9,
U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1,
U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7,
U+1F75->U+03B7, U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF,
U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9,
U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1,
U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1,
U+1F87->U+03B1, U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1,
U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1,
U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7,
U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7,
U+1F97->U+03B7, U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7,
U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7,
U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9,
U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9,
U+1FA7->U+03C9, U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9,
U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9,
U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1,
U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1,
U+1FB8->U+03B1, U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1,
U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7,
U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5,
U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9,
U+1FD1->U+03B9, U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9,
U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9,
U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5,
U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5,
U+1FE7->U+03C5, U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5,
U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9,
U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF,
U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9,
U+0391..U+03A1->U+03B1..U+03C1, U+03B1..U+03C1,
U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB,
U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED,
U+03EF, U+03F3, U+03F8, U+03FB
    html_strip        = 0
}

index downloads_search_delta : downloads_search_main
{
   source            = downloads_search_delta
   path                = /var/lib/sphinxsearch/data/downloads_search_delta
}

source downloads_comments_main : ipb_source_config
{
    # Set our forum PID counter
    sql_query_pre    = REPLACE INTO cache_store VALUES(
'sphinx_downloads_comments_counter', (SELECT max(comment_id) FROM
downloads_comments), 0, UNIX_TIMESTAMP(), 0 )

    # Query posts for the main source
    sql_query        = SELECT c.comment_id, c.comment_id as search_id,
c.comment_mid as comment_member_id, c.comment_date, c.comment_open,
c.comment_text, 
                              f.*, f.file_name as fordinal, cc.field_1 
                      FROM downloads_comments c 
                      LEFT JOIN downloads_files f ON (
c.comment_fid=f.file_id ) 
                      LEFT JOIN downloads_ccontent cc ON
(cc.file_id=f.file_id)

    # Fields
    sql_attr_uint            = search_id
    sql_attr_uint            = file_id
    sql_attr_uint            = file_cat
    sql_attr_uint            = file_open
    sql_attr_uint            = file_views
    sql_attr_uint            = file_rating
    sql_attr_uint            = file_downloads
    sql_attr_uint            = comment_open
    sql_attr_uint            = comment_member_id
    sql_attr_timestamp        = comment_date
    sql_attr_timestamp        = file_updated
    sql_attr_timestamp        = file_submitted
    sql_attr_uint            = file_submitter
    sql_attr_str2ordinal    = fordinal
    sql_attr_float            = file_cost
    sql_ranged_throttle    = 0
}

source downloads_comments_delta : downloads_comments_main
{
    # Override the base sql_query_pre
    sql_query_pre =

    # Query posts for the delta source
    sql_query        = SELECT c.comment_id, c.comment_id as search_id,
c.comment_mid as comment_member_id, c.comment_date, c.comment_open,
c.comment_text, 
                              f.*, f.file_name as fordinal, cc.field_1 
                      FROM downloads_comments c 
                      LEFT JOIN downloads_files f ON (
c.comment_fid=f.file_id ) 
                      LEFT JOIN downloads_ccontent cc ON
(cc.file_id=f.file_id) 
                      WHERE c.comment_id<= ( SELECT cs_value FROM
cache_store WHERE cs_key='sphinx_downloads_comments_counter' )
}

index downloads_comments_main
{
    source            = downloads_comments_main
    path            = /var/lib/sphinxsearch/data/downloads_comments_main

    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len    = 2
#    charset_type    = sbcs
    charset_type    = utf-8
    charset_table = U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z,
U+FF41..U+FF5A->a..z, 0..9, A..Z->a..z, a..z, U+0386->U+03B1,
U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF,
U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9,
U+03AB->U+03C5, U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7,
U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5,
U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2,
U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5,
U+03D5->U+03C6, U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB,
U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3,
U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB,
U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1,
U+03F2->U+03C3, U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5,
U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1,
U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1,
U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1,
U+1F09->U+03B1, U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1,
U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5,
U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5,
U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5,
U+1F1B->U+03B5, U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7,
U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7,
U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7,
U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7,
U+1F2D->U+03B7, U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9,
U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9,
U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9,
U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9,
U+1F3D->U+03B9, U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF,
U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF,
U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF,
U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5,
U+1F51->U+03C5, U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5,
U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5,
U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9,
U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9,
U+1F65->U+03C9, U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9,
U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9,
U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1,
U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7,
U+1F75->U+03B7, U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF,
U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9,
U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1,
U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1,
U+1F87->U+03B1, U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1,
U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1,
U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7,
U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7,
U+1F97->U+03B7, U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7,
U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7,
U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9,
U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9,
U+1FA7->U+03C9, U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9,
U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9,
U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1,
U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1,
U+1FB8->U+03B1, U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1,
U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7,
U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5,
U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9,
U+1FD1->U+03B9, U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9,
U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9,
U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5,
U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5,
U+1FE7->U+03C5, U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5,
U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9,
U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF,
U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9,
U+0391..U+03A1->U+03B1..U+03C1, U+03B1..U+03C1,
U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB,
U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED,
U+03EF, U+03F3, U+03F8, U+03FB
    html_strip        = 0
#infix_fields    = comment_text 
#min_infix_len   = 3 
#enable_star     = 1 
}

index downloads_comments_delta : downloads_comments_main
{
   source            = downloads_comments_delta
   path                =
/var/lib/sphinxsearch/data/downloads_comments_delta
}

################################# --- CLASSIFIEDS ---
##############################
source classifieds_search_ads_main : ipb_source_config
{
    # Set our forum PID counter
    sql_query_pre = SET NAMES utf8
    sql_query_pre    = REPLACE INTO cache_store VALUES(
'sphinx_forums_counter_posts', (SELECT max(classifiedid) FROM
classifieds_classifieds), 0, UNIX_TIMESTAMP(), 0 )

    # Query posts for the main source
    sql_query        = SELECT cc.classifiedid, cc.classifiedid as
search_id, cc.member_id, cc.start_date, REPLACE( cc.description, '-',
'&#8208') as description, cc.categoryid, cc.views, 
                             ca.categoryid, LOWER(ca.title) as
tordinal, REPLACE( ca.title, '-', '&#8208') as title, ca.classifieds,
ca.parentid, ca.slug, 
                            CASE WHEN cc.visible = -1 THEN 1 ELSE 0 END
AS soft_deleted, 
                            CASE WHEN cc.visible = -1 THEN 0 ELSE
cc.visible END AS visible, 
                            CONCAT( SUBSTRING( cc.start_date, 2, 8 ),
'0', LPAD( ca.categoryid, 10, 0 ) ) as last_post_group 
                      FROM classifieds_classifieds cc 
                      LEFT JOIN classifieds_categories ca ON (
cc.categoryid=ca.categoryid )

    # Fields
    sql_attr_uint            = visible
    sql_attr_uint            = soft_deleted
    sql_attr_uint            = search_id
    sql_attr_uint            = categoryid
    sql_attr_timestamp        = start_date
    sql_attr_uint            = member_id
    sql_attr_uint            = views
    sql_attr_str2ordinal    = tordinal
    sql_attr_bigint            = last_post_group
    sql_attr_multi            = uint tag_id from query; SELECT
cc.classifiedid, c.tag_id FROM core_tags c LEFT JOIN
classifieds_classifieds cc ON (cc.classifiedid=c.tag_meta_id) WHERE
c.tag_meta_app='classifieds' AND c.tag_meta_area='ads'

    sql_ranged_throttle    = 0
}

index classifieds_search_ads_main
{
    source            = classifieds_search_ads_main
    path            =
/var/lib/sphinxsearch/data/classifieds_search_ads_main

    docinfo            = extern
    mlock            = 0
    morphology        = none
    min_word_len    = 2
#    charset_type    = sbcs
    charset_type    = utf-8
    charset_table = U+FF10..U+FF19->0..9, U+FF21..U+FF3A->a..z,
U+FF41..U+FF5A->a..z, 0..9, A..Z->a..z, a..z, U+0386->U+03B1,
U+0388->U+03B5, U+0389->U+03B7, U+038A->U+03B9, U+038C->U+03BF,
U+038E->U+03C5, U+038F->U+03C9, U+0390->U+03B9, U+03AA->U+03B9,
U+03AB->U+03C5, U+03AC->U+03B1, U+03AD->U+03B5, U+03AE->U+03B7,
U+03AF->U+03B9, U+03B0->U+03C5, U+03CA->U+03B9, U+03CB->U+03C5,
U+03CC->U+03BF, U+03CD->U+03C5, U+03CE->U+03C9, U+03D0->U+03B2,
U+03D1->U+03B8, U+03D2->U+03C5, U+03D3->U+03C5, U+03D4->U+03C5,
U+03D5->U+03C6, U+03D6->U+03C0, U+03D8->U+03D9, U+03DA->U+03DB,
U+03DC->U+03DD, U+03DE->U+03DF, U+03E0->U+03E1, U+03E2->U+03E3,
U+03E4->U+03E5, U+03E6->U+03E7, U+03E8->U+03E9, U+03EA->U+03EB,
U+03EC->U+03ED, U+03EE->U+03EF, U+03F0->U+03BA, U+03F1->U+03C1,
U+03F2->U+03C3, U+03F4->U+03B8, U+03F5->U+03B5, U+03F6->U+03B5,
U+03F7->U+03F8, U+03F9->U+03C3, U+03FA->U+03FB, U+1F00->U+03B1,
U+1F01->U+03B1, U+1F02->U+03B1, U+1F03->U+03B1, U+1F04->U+03B1,
U+1F05->U+03B1, U+1F06->U+03B1, U+1F07->U+03B1, U+1F08->U+03B1,
U+1F09->U+03B1, U+1F0A->U+03B1, U+1F0B->U+03B1, U+1F0C->U+03B1,
U+1F0D->U+03B1, U+1F0E->U+03B1, U+1F0F->U+03B1, U+1F10->U+03B5,
U+1F11->U+03B5, U+1F12->U+03B5, U+1F13->U+03B5, U+1F14->U+03B5,
U+1F15->U+03B5, U+1F18->U+03B5, U+1F19->U+03B5, U+1F1A->U+03B5,
U+1F1B->U+03B5, U+1F1C->U+03B5, U+1F1D->U+03B5, U+1F20->U+03B7,
U+1F21->U+03B7, U+1F22->U+03B7, U+1F23->U+03B7, U+1F24->U+03B7,
U+1F25->U+03B7, U+1F26->U+03B7, U+1F27->U+03B7, U+1F28->U+03B7,
U+1F29->U+03B7, U+1F2A->U+03B7, U+1F2B->U+03B7, U+1F2C->U+03B7,
U+1F2D->U+03B7, U+1F2E->U+03B7, U+1F2F->U+03B7, U+1F30->U+03B9,
U+1F31->U+03B9, U+1F32->U+03B9, U+1F33->U+03B9, U+1F34->U+03B9,
U+1F35->U+03B9, U+1F36->U+03B9, U+1F37->U+03B9, U+1F38->U+03B9,
U+1F39->U+03B9, U+1F3A->U+03B9, U+1F3B->U+03B9, U+1F3C->U+03B9,
U+1F3D->U+03B9, U+1F3E->U+03B9, U+1F3F->U+03B9, U+1F40->U+03BF,
U+1F41->U+03BF, U+1F42->U+03BF, U+1F43->U+03BF, U+1F44->U+03BF,
U+1F45->U+03BF, U+1F48->U+03BF, U+1F49->U+03BF, U+1F4A->U+03BF,
U+1F4B->U+03BF, U+1F4C->U+03BF, U+1F4D->U+03BF, U+1F50->U+03C5,
U+1F51->U+03C5, U+1F52->U+03C5, U+1F53->U+03C5, U+1F54->U+03C5,
U+1F55->U+03C5, U+1F56->U+03C5, U+1F57->U+03C5, U+1F59->U+03C5,
U+1F5B->U+03C5, U+1F5D->U+03C5, U+1F5F->U+03C5, U+1F60->U+03C9,
U+1F61->U+03C9, U+1F62->U+03C9, U+1F63->U+03C9, U+1F64->U+03C9,
U+1F65->U+03C9, U+1F66->U+03C9, U+1F67->U+03C9, U+1F68->U+03C9,
U+1F69->U+03C9, U+1F6A->U+03C9, U+1F6B->U+03C9, U+1F6C->U+03C9,
U+1F6D->U+03C9, U+1F6E->U+03C9, U+1F6F->U+03C9, U+1F70->U+03B1,
U+1F71->U+03B1, U+1F72->U+03B5, U+1F73->U+03B5, U+1F74->U+03B7,
U+1F75->U+03B7, U+1F76->U+03B9, U+1F77->U+03B9, U+1F78->U+03BF,
U+1F79->U+03BF, U+1F7A->U+03C5, U+1F7B->U+03C5, U+1F7C->U+03C9,
U+1F7D->U+03C9, U+1F80->U+03B1, U+1F81->U+03B1, U+1F82->U+03B1,
U+1F83->U+03B1, U+1F84->U+03B1, U+1F85->U+03B1, U+1F86->U+03B1,
U+1F87->U+03B1, U+1F88->U+03B1, U+1F89->U+03B1, U+1F8A->U+03B1,
U+1F8B->U+03B1, U+1F8C->U+03B1, U+1F8D->U+03B1, U+1F8E->U+03B1,
U+1F8F->U+03B1, U+1F90->U+03B7, U+1F91->U+03B7, U+1F92->U+03B7,
U+1F93->U+03B7, U+1F94->U+03B7, U+1F95->U+03B7, U+1F96->U+03B7,
U+1F97->U+03B7, U+1F98->U+03B7, U+1F99->U+03B7, U+1F9A->U+03B7,
U+1F9B->U+03B7, U+1F9C->U+03B7, U+1F9D->U+03B7, U+1F9E->U+03B7,
U+1F9F->U+03B7, U+1FA0->U+03C9, U+1FA1->U+03C9, U+1FA2->U+03C9,
U+1FA3->U+03C9, U+1FA4->U+03C9, U+1FA5->U+03C9, U+1FA6->U+03C9,
U+1FA7->U+03C9, U+1FA8->U+03C9, U+1FA9->U+03C9, U+1FAA->U+03C9,
U+1FAB->U+03C9, U+1FAC->U+03C9, U+1FAD->U+03C9, U+1FAE->U+03C9,
U+1FAF->U+03C9, U+1FB0->U+03B1, U+1FB1->U+03B1, U+1FB2->U+03B1,
U+1FB3->U+03B1, U+1FB4->U+03B1, U+1FB6->U+03B1, U+1FB7->U+03B1,
U+1FB8->U+03B1, U+1FB9->U+03B1, U+1FBA->U+03B1, U+1FBB->U+03B1,
U+1FBC->U+03B1, U+1FC2->U+03B7, U+1FC3->U+03B7, U+1FC4->U+03B7,
U+1FC6->U+03B7, U+1FC7->U+03B7, U+1FC8->U+03B5, U+1FC9->U+03B5,
U+1FCA->U+03B7, U+1FCB->U+03B7, U+1FCC->U+03B7, U+1FD0->U+03B9,
U+1FD1->U+03B9, U+1FD2->U+03B9, U+1FD3->U+03B9, U+1FD6->U+03B9,
U+1FD7->U+03B9, U+1FD8->U+03B9, U+1FD9->U+03B9, U+1FDA->U+03B9,
U+1FDB->U+03B9, U+1FE0->U+03C5, U+1FE1->U+03C5, U+1FE2->U+03C5,
U+1FE3->U+03C5, U+1FE4->U+03C1, U+1FE5->U+03C1, U+1FE6->U+03C5,
U+1FE7->U+03C5, U+1FE8->U+03C5, U+1FE9->U+03C5, U+1FEA->U+03C5,
U+1FEB->U+03C5, U+1FEC->U+03C1, U+1FF2->U+03C9, U+1FF3->U+03C9,
U+1FF4->U+03C9, U+1FF6->U+03C9, U+1FF7->U+03C9, U+1FF8->U+03BF,
U+1FF9->U+03BF, U+1FFA->U+03C9, U+1FFB->U+03C9, U+1FFC->U+03C9,
U+0391..U+03A1->U+03B1..U+03C1, U+03B1..U+03C1,
U+03A3..U+03A9->U+03C3..U+03C9, U+03C3..U+03C9, U+03C2, U+03D9, U+03DB,
U+03DD, U+03DF, U+03E1, U+03E3, U+03E5, U+03E7, U+03E9, U+03EB, U+03ED,
U+03EF, U+03F3, U+03F8, U+03FB
    html_strip        = 0
#infix_fields    = post, title 
#min_infix_len   = 3 
#enable_star     = 1 
}


#############################################################################
## indexer settings
#############################################################################

indexer
{
    mem_limit            = 256M
}

#############################################################################
## searchd settings
#############################################################################

searchd
{
    listen                = 127.0.0.1
    port                = 3312
    log                    = /var/log/sphinxsearch/searchd.log
    query_log            = /var/log/sphinxsearch/query.log
    read_timeout        = 5
    max_children        = 30
    pid_file            = /var/run/searchd.pid
    max_matches            = 1000
    seamless_rotate        = 0
    preopen_indexes        = 0
    unlink_old            = 1
}

# --eof--



[2]
root@web:~# time /usr/bin/indexer --config
/home/hal9000/sphinx/sphinx.conf --all --rotate
Sphinx 2.0.4-release (r3135)
Copyright © 2001-2012, Andrew Aksyonoff
Copyright © 2008-2012, Sphinx Technologies Inc ( http://sphinxsearch.com)

using config file '/home/hal9000/sphinx/sphinx.conf'...
WARNING: failed to open pid_file '/var/run/searchd.pid'.
indexing index 'core_tags_search_main'...
collected 79 docs, 0.0 MB
sorted 0.0 Mhits, 100.0% done
total 79 docs, 3136 bytes
total 0.007 sec, 407061 bytes/sec, 10254.41 docs/sec
indexing index 'core_tags_search_delta'...
collected 0 docs, 0.0 MB
total 0 docs, 0 bytes
total 0.001 sec, 0 bytes/sec, 0.00 docs/sec
indexing index 'forums_search_posts_main'...
collected 5178581 docs, 3582.3 MB
collected 53 attr values
sorted 0.0 Mvalues, 100.0% done
sorted 403.2 Mhits, 100.0% done
total 5178581 docs, 3582305366 bytes
total 294.098 sec, 12180615 bytes/sec, 17608.29 docs/sec
indexing index 'forums_search_posts_delta'...
collected 4 docs, 0.0 MB
collected 53 attr values
sorted 0.0 Mvalues, 100.0% done
sorted 0.0 Mhits, 100.0% done
total 4 docs, 2022 bytes
total 0.254 sec, 7959 bytes/sec, 15.74 docs/sec
indexing index 'forums_search_archive_main'...
collected 0 docs, 0.0 MB
collected 53 attr values
sorted 0.0 Mvalues, 100.0% done
total 0 docs, 0 bytes
total 0.003 sec, 0 bytes/sec, 0.00 docs/sec
indexing index 'forums_search_archive_delta'...
collected 0 docs, 0.0 MB
collected 53 attr values
sorted 0.0 Mvalues, 100.0% done
total 0 docs, 0 bytes
total 0.002 sec, 0 bytes/sec, 0.00 docs/sec
indexing index 'downloads_search_main'...
collected 418 docs, 0.3 MB
collected 0 attr values
sorted 0.0 Mvalues, 100.0% done
sorted 0.0 Mhits, 100.0% done
total 418 docs, 309909 bytes
total 0.022 sec, 13792122 bytes/sec, 18602.58 docs/sec
indexing index 'downloads_search_delta'...
collected 0 docs, 0.0 MB
collected 0 attr values
sorted 0.0 Mvalues, 100.0% done
total 0 docs, 0 bytes
total 0.001 sec, 0 bytes/sec, 0.00 docs/sec
indexing index 'downloads_comments_main'...
collected 476 docs, 0.6 MB
sorted 0.0 Mhits, 100.0% done
total 476 docs, 554162 bytes
total 0.028 sec, 19287946 bytes/sec, 16567.47 docs/sec
indexing index 'downloads_comments_delta'...
collected 476 docs, 0.6 MB
sorted 0.0 Mhits, 100.0% done
total 476 docs, 554162 bytes
total 0.028 sec, 19212383 bytes/sec, 16502.56 docs/sec
indexing index 'classifieds_search_ads_main'...
collected 117446 docs, 75.8 MB
collected 8 attr values
sorted 0.0 Mvalues, 100.0% done
sorted 8.2 Mhits, 100.0% done
total 117446 docs, 75849763 bytes
total 4.900 sec, 15479401 bytes/sec, 23968.35 docs/sec
total 603 reads, 1.552 sec, 3352.4 kb/call avg, 2.5 msec/call avg
total 4332 writes, 3.103 sec, 909.7 kb/call avg, 0.7 msec/call avg
WARNING: indices NOT rotated.

real    4m59.394s
user    4m8.060s
sys    0m7.152s


[3]
root@web:~# time /usr/bin/indexer --config
/home/hal9000/sphinx/sphinx.conf core_tags_search_delta
forums_search_posts_delta forums_search_archive_delta
downloads_search_delta downloads_comments_delta --rotate
Sphinx 2.0.4-release (r3135)
Copyright © 2001-2012, Andrew Aksyonoff
Copyright © 2008-2012, Sphinx Technologies Inc (http://sphinxsearch.com)

using config file '/home/hal9000/sphinx/sphinx.conf'...
WARNING: failed to open pid_file '/var/run/searchd.pid'.
indexing index 'core_tags_search_delta'...
collected 0 docs, 0.0 MB
total 0 docs, 0 bytes
total 0.004 sec, 0 bytes/sec, 0.00 docs/sec
indexing index 'forums_search_posts_delta'...
collected 5059226 docs, 3509.2 MB
collected 53 attr values
sorted 0.0 Mvalues, 100.0% done
sorted 392.9 Mhits, 100.0% done
total 5059226 docs, 3509249824 bytes
total 253.783 sec, 13827722 bytes/sec, 19935.19 docs/sec
indexing index 'forums_search_archive_delta'...
collected 0 docs, 0.0 MB
collected 53 attr values
sorted 0.0 Mvalues, 100.0% done
total 0 docs, 0 bytes
total 0.003 sec, 0 bytes/sec, 0.00 docs/sec
indexing index 'downloads_search_delta'...
collected 0 docs, 0.0 MB
collected 0 attr values
sorted 0.0 Mvalues, 100.0% done
total 0 docs, 0 bytes
total 0.002 sec, 0 bytes/sec, 0.00 docs/sec
indexing index 'downloads_comments_delta'...
collected 476 docs, 0.6 MB
sorted 0.0 Mhits, 100.0% done
total 476 docs, 554162 bytes
total 0.029 sec, 18944414 bytes/sec, 16272.39 docs/sec
total 474 reads, 1.395 sec, 4095.7 kb/call avg, 2.9 msec/call avg
total 4097 writes, 2.650 sec, 931.9 kb/call avg, 0.6 msec/call avg
WARNING: indices NOT rotated.

real    4m13.844s
user    3m32.681s
sys    0m6.336s

Thank you

Well-Trained Mind Press · January 17, 2013

How many posts do you have? Our forum of about 4mil takes about this long on the indexer --all --rotate but we run it on an external amazon m1.medium server so it doesn't bog our site down.

Have you considered trying to run it of an amazon m1.small or the free m1.micro server to see how that would work?

John

iliasp · January 17, 2013

I think that he has more than 5 millions

GreenLinks · January 18, 2013

Logically Sphinx indexes should be built on separate tables. Right now all post data is stored on one index which tends to become a huge index for big boards. Our forums_search_posts_main index is 24685MB . Indexes will never be updated fast with the current method. We are working on specifications regarding our boards search needs and will highly possible customise IPB search and indexing period.

Axivo Inc does magical things with their Searchlight product , unfortunately it is only available to vBulletin 3.8

However with some basic sphinx configuration , i reduced primary preaching time on indexer.

Default IPB values

Starting searchd: Sphinx 2.0.5-id64-release (r3308)
Copyright © 2001-2012, Andrew Aksyonoff
Copyright © 2008-2012, Sphinx Technologies Inc (http://sphinxsearch.com)

using config file '/etc/sphinx/sphinx.conf'...
WARNING: compat_sphinxql_magics=1 is deprecated; please update your application and config
listening on 127.0.0.1:9312
listening on all interfaces, port=3312
precaching index 'core_tags_search_main'
rotating index 'core_tags_search_main': success             
precaching index 'core_tags_search_delta'
rotating index 'core_tags_search_delta': success            
precaching index 'forums_search_posts_main'
rotating index 'forums_search_posts_main': success          
precaching index 'forums_search_posts_delta'
precaching index 'forums_search_archive_main'               
precaching index 'forums_search_archive_delta'              
precaching index 'blog_search_main'                         
precaching index 'blog_search_delta'                        
precaching index 'blog_comments_main'                       
rotating index 'blog_comments_main': success                
precaching index 'blog_comments_delta'
rotating index 'blog_comments_delta': success               
precached 10 indexes in 210.836 sec

My customised configuration

Starting searchd: Sphinx 2.0.5-id64-release (r3308)
Copyright © 2001-2012, Andrew Aksyonoff
Copyright © 2008-2012, Sphinx Technologies Inc (http://sphinxsearch.com)

using config file '/etc/sphinx/sphinx.conf'...
listening on 127.0.0.1:9312
listening on all interfaces, port=3312
precaching index 'core_tags_search_main'
precaching index 'core_tags_search_delta'                   
precaching index 'forums_search_posts_main'                 
precaching index 'forums_search_posts_delta'                
precaching index 'forums_search_archive_main'               
precaching index 'forums_search_archive_delta'              
precaching index 'blog_search_main'                         
precaching index 'blog_search_delta'                        
precaching index 'blog_comments_main'                       
precaching index 'blog_comments_delta'                      
rotating index 'blog_comments_delta': success               
precached 10 indexes in 58.230 sec

Will edit the post later on and add total index rotating time for 15 mil post

GreenLinks · January 18, 2013

Basically IPB should offer sql_query_range and sql_range_step so that table locks will be eliminated during indexing period. This will allow multiple index support and reduce table locks during reindexing period. Even with InnoDB it is expected to face table locks when you query tables with over 2-3 million rows.

Invision Community 5: Assign topics to moderators

Invision Community 4: Pages databases in Clubs

Invision Community 5: Live Topic Improvements

Invision Community 5: New Live Community Features

Sphinx Problem

Recommended Posts

Real Hal9000

Link to comment

Share on other sites

Well-Trained Mind Press

Link to comment

Share on other sites

iliasp

Link to comment

Share on other sites

GreenLinks

Link to comment

Share on other sites

GreenLinks

Link to comment

Share on other sites

Archived

Recently Browsing 0 members

Upcoming Events

Trending Content

More