ES实战--mapping.json 和 populate.sh 适配es8

#mapping.json
{
  "settings": {
    "number_of_shards": 2,
    "number_of_replicas": 1,
    "analysis": {
      "analyzer": {
        "myCustomAnalyzer": {
          "type": "custom",
          "tokenizer": "myCustomTokenizer",
          "filter": ["myCustomFilter1", "myCustomFilter2"],
          "char_filter": ["myCustomCharFilter"]
        }
      },
      "tokenizer": {
        "myCustomTokenizer": {
          "type": "letter"
        },
        "myCustomNGramTokenizer": {
          "type": "ngram",
          "min_gram": 2,
          "max_gram": 3
        }
      },
      "filter": {
        "myCustomFilter1": {
          "type": "lowercase"
        },
        "myCustomFilter2": {
          "type": "kstem"
        }
      },
      "char_filter": {
        "myCustomCharFilter": {
          "type": "mapping",
          "mappings": ["ph=>f", "u=>you", "ES=>Elasticsearch"]
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "date": {
        "type": "date",
        "format": "date_hour_minute"
      },
      "reviews": {
        "type": "integer",
        "null_value": 0
      },
      "attendees": {
        "type": "keyword"
      },
      "location_event": {
        "properties": {
          "name": {
            "type": "text"
          },
          "geolocation": {
            "type": "geo_point"
          }
        }
      },
      "host": {
        "type": "text"
      },
      "description": {
        "type": "text"
      },
      "title": {
        "type": "text"
      },
      "organizer": {
        "type": "text"
      },
      "name": {
        "type": "text"
      },
      "created_on": {
        "type": "date",
        "format": "yyyy-MM-dd"
      },
      "tags": {
        "type": "text",
        "fields": {
          "verbatim": {
            "type": "keyword"
          }
        }
      },
      "members": {
        "type": "text"
      },
      "location_group": {
        "type": "text"
      }
    }
  }
}

#populate.sh
#!/usr/bin/env bash

ADDRESS=$1

if [ -z $ADDRESS ]; then
  ADDRESS="localhost:9200"
fi

# Check that Elasticsearch is running
curl -s "http://$ADDRESS" 2>&1 > /dev/null
if [ $? != 0 ]; then
    echo "Unable to contact Elasticsearch at $ADDRESS"
    echo "Please ensure Elasticsearch is running and can be reached at http://$ADDRESS/"
    exit -1
fi

echo "WARNING, this script will delete the 'get-together' and the 'myindex' indices and re-index all data!"
echo "Press Control-C to cancel this operation."
echo
echo "Press [Enter] to continue."
read

# Delete the old index, swallow failures if it doesn't exist
curl -s -XDELETE "$ADDRESS/get-together" > /dev/null

# Create the next index using mapping.json
echo "Creating 'get-together' index..."
curl -s -XPUT "$ADDRESS/get-together" -H "Content-Type: application/json" -d@$(dirname $0)/mapping.json

# Wait for index to become yellow
curl -s "$ADDRESS/get-together/_health?wait_for_status=yellow&timeout=10s" > /dev/null
echo
echo "Done creating 'get-together' index."

echo
echo "Indexing data..."

echo "Indexing groups..."
curl -s -XPUT "$ADDRESS/get-together/_doc/1" -H "Content-Type: application/json" -d'{
  "name": "Denver Clojure",
  "organizer": ["Daniel", "Lee"],
  "description": "Group of Clojure enthusiasts from Denver who want to hack on code together and learn more about Clojure",
  "created_on": "2012-06-15",
  "tags": ["clojure", "denver", "functional programming", "jvm", "java"],
  "members": ["Lee", "Daniel", "Mike"],
  "location_group": "Denver, Colorado, USA"
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/2" -H "Content-Type: application/json" -d'{
  "name": "Elasticsearch Denver",
  "organizer": "Lee",
  "description": "Get together to learn more about using Elasticsearch, the applications and neat things you can do with ES!",
  "created_on": "2013-03-15",
  "tags": ["denver", "elasticsearch", "big data", "lucene", "solr"],
  "members": ["Lee", "Mike"],
  "location_group": "Denver, Colorado, USA"
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/3" -H "Content-Type: application/json"  -d'{
  "name": "Elasticsearch San Francisco",
  "organizer": "Mik",
  "description": "Elasticsearch group for ES users of all knowledge levels",
  "created_on": "2012-08-07",
  "tags": ["elasticsearch", "big data", "lucene", "open source"],
  "members": ["Lee", "Igor"],
  "location_group": "San Francisco, California, USA"
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/4" -H "Content-Type: application/json"  -d'{
  "name": "Boulder/Denver big data get-together",
  "organizer": "Andy",
  "description": "Come learn and share your experience with nosql & big data technologies, no experience required",
  "created_on": "2010-04-02",
  "tags": ["big data", "data visualization", "open source", "cloud computing", "hadoop"],
  "members": ["Greg", "Bill"],
  "location_group": "Boulder, Colorado, USA"
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/5" -H "Content-Type: application/json"  -d'{
  "name": "Enterprise search London get-together",
  "organizer": "Tyler",
  "description": "Enterprise search get-togethers are an opportunity to get together with other people doing search.",
  "created_on": "2009-11-25",
  "tags": ["enterprise search", "apache lucene", "solr", "open source", "text analytics"],
  "members": ["Clint", "James"],
  "location_group": "London, England, UK"
}'

echo
echo "Done indexing groups."

echo "Indexing events..."

curl -s -XPUT "$ADDRESS/get-together/_doc/100?routing=1" -H "Content-Type: application/json"   -d'{
  "host": ["Lee", "Troy"],
  "title": "Liberator and Immutant",
  "description": "We will discuss two different frameworks in Clojure for doing different things. Liberator is a ring-compatible web framework based on Erlang Webmachine. Immutant is an all-in-one enterprise application based on JBoss.",
  "attendees": ["Lee", "Troy", "Daniel", "Tom"],
  "date": "2013-09-05T18:00",
  "location_event": {
    "name": "Stoneys Full Steam Tavern",
    "geolocation": "39.752337,-105.00083"
  },
  "reviews": 4
}'
echo
curl -s -XPUT "$ADDRESS/get-together/_doc/101?routing=1" -H "Content-Type: application/json"  -d'{
  "host": "Sean",
  "title": "Sunday, Surly Sunday",
  "description": "Sort out any setup issues and work on Surlybird issues. We can use the EC2 node as a bounce point for pairing.",
  "attendees": ["Daniel", "Michael", "Sean"],
  "date": "2013-07-21T18:30",
  "location_event": {
    "name": "IRC, #denofclojure"
  },
  "reviews": 2
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/102?routing=1" -H "Content-Type: application/json"   -d'{
  "host": "Daniel",
  "title": "10 Clojure coding techniques you should know, and project openbike",
  "description": "What are ten Clojure coding techniques that you wish everyone knew? We will also check on the status of Project Openbike.",
  "attendees": ["Lee", "Tyler", "Daniel", "Stuart", "Lance"],
  "date": "2013-07-11T18:00",
  "location_event": {
    "name": "Stoneys Full Steam Tavern",
    "geolocation": "39.752337,-105.00083"
  },
  "reviews": 3
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/103?routing=2" -H "Content-Type: application/json"  -d'{
  "host": "Lee",
  "title": "Introduction to Elasticsearch",
  "description": "An introduction to ES and each other. We can meet and greet and I will present on some Elasticsearch basics and how we use it.",
  "attendees": ["Lee", "Martin", "Greg", "Mike"],
  "date": "2013-04-17T19:00",
  "location_event": {
    "name": "Stoneys Full Steam Tavern",
    "geolocation": "39.752337,-105.00083"
  },
  "reviews": 5
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/104?routing=2" -H "Content-Type: application/json"   -d'{
  "host": "Lee",
  "title": "Queries and Filters",
  "description": "A get together to talk about different ways to query Elasticsearch, what works best for different kinds of applications.",
  "attendees": ["Lee", "Greg", "Richard"],
  "date": "2013-06-17T18:00",
  "location_event": {
    "name": "Stoneys Full Steam Tavern",
    "geolocation": "39.752337,-105.00083"
  },
  "reviews": 1
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/105?routing=2" -H "Content-Type: application/json"   -d'{
  "host": "Lee",
  "title": "Elasticsearch and Logstash",
  "description": "We can get together and talk about Logstash - http://logstash.net with a sneak peek at Kibana",
  "attendees": ["Lee", "Greg", "Mike", "Delilah"],
  "date": "2013-07-17T18:30",
  "location_event": {
    "name": "Stoneys Full Steam Tavern",
    "geolocation": "39.752337,-105.00083"
  },
  "reviews": null
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/106?routing=3" -H "Content-Type: application/json"   -d'{
  "host": "Mik",
  "title": "Social management and monitoring tools",
  "description": "Shay Banon will be there to answer questions and we can talk about management tools.",
  "attendees": ["Shay", "Mik", "John", "Chris"],
  "date": "2013-03-06T18:00",
  "location_event": {
    "name": "Quid Inc",
    "geolocation": "37.798442,-122.399801"
  },
  "reviews": 5
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/107?routing=3" -H "Content-Type: application/json"   -d'{
  "host": "Mik",
  "title": "Logging and Elasticsearch",
  "description": "Get a deep dive for what Elasticsearch is and how it can be used for logging with Logstash as well as Kibana!",
  "attendees": ["Shay", "Rashid", "Erik", "Grant", "Mik"],
  "date": "2013-04-08T18:00",
  "location_event": {
    "name": "Salesforce headquarters",
    "geolocation": "37.793592,-122.397033"
  },
  "reviews": 3
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/108?routing=3" -H "Content-Type: application/json"   -d'{
  "host": "Elyse",
  "title": "Piggyback on Elasticsearch training in San Francisco",
  "description": "We can piggyback on training by Elasticsearch to have some Q&A time with the ES devs",
  "attendees": ["Shay", "Igor", "Uri", "Elyse"],
  "date": "2013-05-23T19:00",
  "location_event": {
    "name": "NoSQL Roadshow",
    "geolocation": "37.787742,-122.398964"
  },
  "reviews": 5
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/109?routing=4" -H "Content-Type: application/json"  -d'{
  "host": "Andy",
  "title": "Hortonworks, the future of Hadoop and big data",
  "description": "Presentation on the work that hortonworks is doing on Hadoop",
  "attendees": ["Andy", "Simon", "David", "Sam"],
  "date": "2013-06-19T18:00",
  "location_event": {
    "name": "SendGrid Denver office",
    "geolocation": "39.748477,-104.998852"
  },
  "reviews": 2
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/110?routing=4" -H "Content-Type: application/json"  -d'{ 
  "host": "Andy",
  "title": "Big Data and the cloud at Microsoft",
  "description": "Discussion about the Microsoft Azure cloud and HDInsight.",
  "attendees": ["Andy", "Michael", "Ben", "David"],
  "date": "2013-07-31T18:00",
  "location_event": {
    "name": "Bing Boulder office",
    "geolocation": "40.018528,-105.275806"
  },
  "reviews": 1
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/111?routing=4" -H "Content-Type: application/json"  -d'{
  "host": "Andy",
  "title": "Moving Hadoop to the mainstream",
  "description": "Come hear about how Hadoop is moving to the main stream",
  "attendees": ["Andy", "Matt", "Bill"],
  "date": "2013-07-21T18:00",
  "location_event": {
    "name": "Courtyard Boulder Louisville",
    "geolocation": "39.959409,-105.163497"
  },
  "reviews": 4
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/112?routing=5" -H "Content-Type: application/json"  -d'{
  "host": "Dave Nolan",
  "title": "real-time Elasticsearch",
  "description": "We will discuss using Elasticsearch to index data in real time",
  "attendees": ["Dave", "Shay", "John", "Harry"],
  "date": "2013-02-18T18:30",
  "location_event": {
    "name": "SkillsMatter Exchange",
    "geolocation": "51.524806,-0.099095"
  },
  "reviews": 3
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/113?routing=5" -H "Content-Type: application/json"   -d'{
  "host": "Dave",
  "title": "Elasticsearch at Rangespan and Exonar",
  "description": "Representatives from Rangespan and Exonar will come and discuss how they use Elasticsearch",
  "attendees": ["Dave", "Andrew", "David", "Clint"],
  "date": "2013-06-24T18:30",
  "location_event": {
    "name": "Alumni Theatre",
    "geolocation": "51.51558,-0.117699"
  },
  "reviews": 3
}'

echo
curl -s -XPUT "$ADDRESS/get-together/_doc/114?routing=5" -H "Content-Type: application/json"  -d'{
  "host": "Yann",
  "title": "Using Hadoop with Elasticsearch",
  "description": "We will walk through using Hadoop with Elasticsearch for big data crunching!",
  "attendees": ["Yann", "Bill", "James"],
  "date": "2013-09-09T18:30",
  "location_event": {
    "name": "SkillsMatter Exchange",
    "geolocation": "51.524806,-0.099095"
  },
  "reviews": 2
}'

echo
echo "Done indexing events."

# Refresh so data is available
curl -s -XPOST "$ADDRESS/get-together/_refresh"

echo
echo "Done indexing data."
echo

echo
echo "Creating Templates."
curl -s -XPUT "http://$ADDRESS/_index_template/logging_index_all" -H "Content-Type: application/json" -d'{
    "index_patterns" : ["logstash-09-*"],
    "priority" : 1,
    "template":{
    "settings" : {
        "number_of_shards" : 2,
        "number_of_replicas" : 1
   },
    "mappings" : {
    	"properties": {
        	"date": { "type": "date", "store": false }
     	 }
    },
    "aliases" : { "november" : {} }
    }
}'

echo
curl -s -XPUT "http://$ADDRESS/_index_template/logging_index" -H "Content-Type: application/json"  -d '{
    "index_patterns" : ["logstash-*"],
    "priority" : 0,
    "template":{
    	"settings" : {
        	"number_of_shards" : 2,
        	"number_of_replicas" : 1
   	},
    "mappings" : {
         "properties": {
        "date": { "type": "date", "store": true }
      }
    }
   }
}'
echo
echo "Done Creating Templates."


echo
echo "Adding Dynamic Mapping"
curl -s -XDELETE "http://$ADDRESS/myindex" > /dev/null
curl -s -XPUT "http://$ADDRESS/myindex" -H "Content-Type: application/json"  -d'
{
  "mappings": {
    "dynamic_templates": [
      {
        "UUID": {
          "match": "*_guid",
          "match_mapping_type": "string",
          "mapping": {
            "type": "keyword"
          }
        }
      }
    ]
  }
}'

echo
echo "Done Adding Dynamic Mapping"

echo
echo "Adding Aliases"
curl -s -XDELETE "http://$ADDRESS/november_2014_invoices" > /dev/null
curl -s -XDELETE "http://$ADDRESS/december_2014_invoices" > /dev/null
curl -s -XPUT "http://$ADDRESS/november_2014_invoices" -H "Content-Type: application/json" -d'{}'
echo
curl -s -XPUT "http://$ADDRESS/december_2014_invoices" -H "Content-Type: application/json"   -d'
{
    "mappings" :
    {
            "properties" :
            {
                "revenue" : { "type" : "integer" }
            }
    }
}'

echo
curl -s -XPOST "http://$ADDRESS/_aliases" -H "Content-Type: application/json" -d'
{
    "actions" : [
        {
            "add": {
                "index": "november_2014_invoices",
                "alias": "2014_invoices"
            }
        },
        {
            "add": {
                "index": "december_2014_invoices",
                "alias": "2014_invoices"
            }
        },
        {
            "remove": {
                "index": "myindex",
                "alias": "december_2014_invoices"
            }
        }
    ]
}'

echo
echo "Done Adding Aliases"

echo "Adding Filter Alias"
curl -s -XPOST "http://$ADDRESS/_aliases" -H "Content-Type: application/json"   -d '
{
    "actions" : [
        {
            "add" : {
                 "index" : "december_2014_invoices",
                 "alias" : "bigmoney",
                 "filter" :
                 {
                    "range" :
                    {
                      "revenue" :
                      {
                        "gt" : 1000
                      }

                    }
                 }
            }
        }
    ]
}'
echo
echo "Done Adding Filter Alias"

echo
echo "Adding Routing Alias"
curl -s -XPOST "http://$ADDRESS/_aliases" -H "Content-Type: application/json"  -d '
{
    "actions" : [
        {
            "add" : {
                 "index" : "december_2014_invoices",
                 "alias" : "2014_invoices",
                 "search_routing" : "en,es",
                 "index_routing" : "en"
            }
        }
    ]
}'
echo
echo "Done Adding Routing Alias"

echo