• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    迪恩网络公众号

Python control.start_map函数代码示例

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

本文整理汇总了Python中mapreduce.control.start_map函数的典型用法代码示例。如果您正苦于以下问题:Python start_map函数的具体用法?Python start_map怎么用?Python start_map使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。



在下文中一共展示了start_map函数的20个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于我们的系统推荐出更棒的Python代码示例。

示例1: get

 def get(self):
     mapreduce_params = {
           'entity_kind': 'models.Topic',
     }
     
     control.start_map("DeleteOldUpdates", "mapjob.keep_thirty_updates", "mapreduce.input_readers.DatastoreInputReader", mapreduce_params, 2)
     self.response.out.write("ok")
开发者ID:ThrowDice,项目名称:Reader2Twitter,代码行数:7,代码来源:admin.py


示例2: get

 def get(self):
     # TODO(qyearsley): Add test coverage. See catapult:#1346.
     name = "Update test deprecation status."
     handler = "dashboard.mr.DeprecateTestsMapper"
     reader = "mapreduce.input_readers.DatastoreInputReader"
     mapper_parameters = {"entity_kind": ("dashboard.models.graph_data.TestMetadata"), "filters": []}
     mr_control.start_map(name, handler, reader, mapper_parameters)
开发者ID:n054,项目名称:catapult,代码行数:7,代码来源:mr.py


示例3: _run_test

  def _run_test(self, num_shards, num_files):
    bucket_name = "testing"
    object_prefix = "file-"
    job_name = "test_map"
    input_class = (input_readers.__name__ + "." +
                   input_readers._GoogleCloudStorageInputReader.__name__)

    expected_content = self.create_test_content(bucket_name,
                                                object_prefix,
                                                num_files)

    control.start_map(
        job_name,
        __name__ + "." + "_input_reader_memory_mapper",
        input_class,
        {
            "input_reader": {
                "bucket_name": bucket_name,
                "objects": [object_prefix + "*"]
            },
        },
        shard_count=num_shards)

    test_support.execute_until_empty(self.taskqueue)
    self.assertEqual(expected_content.sort(), _memory_mapper_data.sort())
开发者ID:Karima-Kaddouri,项目名称:appengine-mapreduce,代码行数:25,代码来源:input_readers_end_to_end_test.py


示例4: testHandlerSerialization

    def testHandlerSerialization(self):
        """Test serializable handler works with MR and shard retry."""
        entity_count = 10

        for _ in range(entity_count):
            TestEntity(int_property=-1).put()

        # Force handler to serialize on every call.
        handlers._SLICE_DURATION_SEC = 0

        control.start_map(
            "test_map",
            __name__ + ".SerializableHandler",
            "mapreduce.input_readers.DatastoreInputReader",
            {"entity_kind": __name__ + "." + TestEntity.__name__},
            shard_count=1,
            base_path="/mapreduce_base_path",
        )

        task_run_counts = test_support.execute_until_empty(self.taskqueue)
        self.assertEquals(
            task_run_counts[handlers.MapperWorkerCallbackHandler],
            entity_count + 1 + SerializableHandler.TASKS_CONSUMED_BY_RETRY,
        )
        vals = [e.int_property for e in TestEntity.all()]
        vals.sort()
        # SerializableHandler updates int_property to be incremental from 0 to 9.
        self.assertEquals(range(10), vals)
开发者ID:johnwlockwood,项目名称:appengine-mapreduce,代码行数:28,代码来源:end_to_end_test.py


示例5: start_map

def start_map(
    fbl,
    name,
    handler_spec,
    entity_kind,
    filters=None,
    handle_batch_size=None,
    output_writer_spec=None,
    output_writer=None,
    queue='slow-queue',
    extra_mapper_params=None,
    randomize_tokens=True,
):
    filters = filters or []
    output_writer = output_writer or {}
    extra_mapper_params = extra_mapper_params or {}
    mapper_params = {
        'entity_kind': entity_kind,
        'handle_batch_size': handle_batch_size,
        'filters': filters,
        'output_writer': output_writer,
    }
    mapper_params.update(get_fblookup_params(fbl, randomize_tokens=randomize_tokens))
    mapper_params.update(extra_mapper_params)
    control.start_map(
        name=name,
        reader_spec='mapreduce.input_readers.DatastoreInputReader',
        handler_spec=handler_spec,
        output_writer_spec=output_writer_spec,
        shard_count=16,  # since we want to stick it in the slow-queue, and don't care how fast it executes
        queue_name=queue,
        mapper_parameters=mapper_params,
    )
开发者ID:mikelambert,项目名称:dancedeets-monorepo,代码行数:33,代码来源:fb_mapreduce.py


示例6: testHandlerSerialization

  def testHandlerSerialization(self):
    """Test serializable handler works with MR and shard retry."""
    entity_count = 10

    for _ in range(entity_count):
      TestEntity(int_property=-1).put()

    # Force handler to serialize on every call.
    parameters.config._SLICE_DURATION_SEC = 0

    control.start_map(
        "test_map",
        __name__ + ".SerializableHandler",
        "mapreduce.input_readers.DatastoreInputReader",
        {
            "entity_kind": __name__ + "." + TestEntity.__name__,
        },
        shard_count=1,
        base_path="/mapreduce_base_path")

    task_run_counts = test_support.execute_until_empty(self.taskqueue)
    self.assertEquals(
        task_run_counts[handlers.MapperWorkerCallbackHandler],
        # Shard retries + one per entity + one to exhaust input reader + one for
        # finalization.
        SerializableHandler.FAILURES_INDUCED_BY_INSTANCE + entity_count + 1 + 1)
    vals = [e.int_property for e in TestEntity.all()]
    vals.sort()
    # SerializableHandler updates int_property to be incremental from 0 to 9.
    self.assertEquals(range(10), vals)
开发者ID:Karima-Kaddouri,项目名称:appengine-mapreduce,代码行数:30,代码来源:end_to_end_test.py


示例7: get

 def get(self):
     control.start_map(
         name='Compute User-Event Stats',
         reader_spec='mapreduce.input_readers.DatastoreInputReader',
         handler_spec='users.user_event_tasks.map_compute_user_stats',
         mapper_parameters={'entity_kind': 'users.users.User'},
         queue_name='fast-queue',
         shard_count=5,
     )
开发者ID:DanceDeets,项目名称:dancedeets-server,代码行数:9,代码来源:user_event_tasks.py


示例8: get

 def get(self):
   name = 'Update anomalies with units.'
   handler = ('dashboard.mr.StoreUnitsInAnomalyEntity')
   reader = 'mapreduce.input_readers.DatastoreInputReader'
   mapper_parameters = {
       'entity_kind': ('dashboard.models.graph_data.Anomaly'),
       'filters': [],
   }
   mr_control.start_map(name, handler, reader, mapper_parameters)
开发者ID:joelagnel,项目名称:catapult,代码行数:9,代码来源:mr.py


示例9: get

 def get(self):
     table = self.request.get('table') # users.users.User or events.eventdata.DBEvent or ...
     control.start_map(
         name='Resave %s' % table,
         reader_spec='mapreduce.input_readers.DatastoreInputReader',
         handler_spec='servlets.tools.resave_table',
         mapper_parameters={
             'entity_kind': table,
         },
     )
开发者ID:DanceDeets,项目名称:dancedeets-server,代码行数:10,代码来源:tools.py


示例10: get

 def get(self):
     control.start_map(
         name='Fixup Events',
         reader_spec='mapreduce.input_readers.DatastoreInputReader',
         handler_spec='dancedeets.servlets.tools.resave_object',
         mapper_parameters={
             'entity_kind': 'dancedeets.events.eventdata.DBEvent',
         },
         shard_count=16,
     )
开发者ID:mikelambert,项目名称:dancedeets-monorepo,代码行数:10,代码来源:tools.py


示例11: get

 def get(self):
     # TODO(qyearsley): Add test coverage. See catapult:#1346.
     name = 'Update test deprecation status.'
     handler = ('dashboard.mr.DeprecateTestsMapper')
     reader = 'mapreduce.input_readers.DatastoreInputReader'
     mapper_parameters = {
         'entity_kind': ('dashboard.models.graph_data.TestMetadata'),
         'filters': [('has_rows', '=', True), ('deprecated', '=', False)],
     }
     mr_control.start_map(name, handler, reader, mapper_parameters)
开发者ID:endlessm,项目名称:chromium-browser,代码行数:10,代码来源:mr.py


示例12: delete_all_questions

def delete_all_questions():
    logging.info("Delete all existing questions")
    ctrl.start_map("Delete all Question entities", 
                   'locql.delete_entity', 
                   'mapreduce.input_readers.DatastoreKeyInputReader', 
                   {'entity_kind': 'locql.Question'})
    ctrl.start_map("Delete all TermStat entities", 
                   'locql.delete_entity', 
                   'mapreduce.input_readers.DatastoreKeyInputReader',
                   {'entity_kind': 'locql.TermStat'})
开发者ID:Discoverful,项目名称:LOCQL-Research,代码行数:10,代码来源:locql.py


示例13: begin_user_ranking_calculations

def begin_user_ranking_calculations():
    control.start_map(
        name='Compute City Rankings by Users',
        reader_spec='mapreduce.input_readers.DatastoreInputReader',
        handler_spec='dancedeets.rankings.rankings.count_user_for_city',
        mapper_parameters={'entity_kind': 'dancedeets.users.users.User'},
        queue_name='fast-queue',
        shard_count=16,
        _app=USER_FOR_CITY_RANKING,
    )
    _compute_summary(expiry=5 * 60)  # 5 minutes
开发者ID:mikelambert,项目名称:dancedeets-monorepo,代码行数:11,代码来源:rankings.py


示例14: get

 def get(self):
   mr_control.start_map(
    self.request.get("name"),
    self.request.get("reader_spec", "your_mapreduce.map"),
    self.request.get("reader_parameters",
                     "mapreduce.input_readers.DatastoreInputReader"),
    { "entity_kind": self.request.get("entity_kind", "models.YourModel"),
      "processing_rate": int(self.request.get("processing_rate", 100)) },
    mapreduce_parameters={"done_callback": self.request.get("done_callback",
                                                            None) } )
   self.response.out.write("MapReduce scheduled");
开发者ID:kalinr,项目名称:Wacky-Words-Facebook-MMO,代码行数:11,代码来源:cron_mapreduce.py


示例15: start_map

def start_map(name, params=None, eta=None, countdown=None):
  for config in status.MapReduceYaml.to_dict(status.get_mapreduce_yaml()):
    if config.get('name') == name:
      break

  # Add the mapreduce specific parameters to the params dictionary
  config['mapper_params'].update(params if params else {})

  control.start_map(config['name'], config['mapper_handler'],
                    config['mapper_input_reader'], config['mapper_params'],
                    eta=eta,
                    countdown=countdown)
开发者ID:rhyolight,项目名称:nupic.son,代码行数:12,代码来源:control.py


示例16: get

    def get(self):
        processing_rate = 3
        shard_count = 2

        control.start_map(
            "Iterate over all Dummy objects in the DB",  # this an arbitrary description string
            "tasks.mapper_function",  # this is the function that will bne
            "mapreduce.input_readers.DatastoreInputReader",
            {
                "entity_kind": "dummy.Dummy",  # the model that you will iterate over
                "processing_rate": processing_rate,  # how many entities will each shard process
            },
            shard_count=shard_count,  # how many shards will be created by every MR controller
            queue_name="default",  # the name of the queue that will be used for this MR's jobs, I used default to minimize config
        )
开发者ID:yuri-wisestamp,项目名称:gae-mr-example,代码行数:15,代码来源:tasks.py


示例17: post

 def post(self):
   """ Generate data sets here """
   if self.request.get("generate"):
     # For SDK only generate 1k and less 
     num_entries = int(self.request.get("num_entries"))
     user = self.request.get("user")
     name = self.request.get("name")
     char_per_word = int(self.request.get("char_per_word"))
     entries_pp = int(self.request.get("entries_per_pipe"))
     route = gen_data(num_entries, user, name, char_per_word, entries_pp) 
     self.redirect('/data/wc')
     # pipeline seems broken
     #self.redirect(route)
   elif self.request.get("delete"):
     name = self.request.get("name") 
     dataset = WCDataSet.get_by_key_name(name)
     num_entries = dataset.num_entries
     mapreduce_id = control.start_map(
           name="Word removal",
           handler_spec="data.wordcount.delete_dataset",
           reader_spec="mapreduce.input_readers.DatastoreInputReader",
           mapper_parameters={
               "entity_kind": "data.wordcount." + get_word_class(num_entries),
               "processing_rate": 200
           },
           shard_count=64,
           mapreduce_parameters={model.MapreduceSpec.PARAM_DONE_CALLBACK: 
                      '/data/wc/delete_callback'},
           queue_name="default",
         )
     dataset.state = "Deleting" 
     dataset.mr_id = mapreduce_id
     dataset.put()
     self.redirect('/data/wc')
开发者ID:nlake44,项目名称:GAEBenchmark,代码行数:34,代码来源:wordcount.py


示例18: testStartMap_RaisingHooks

  def testStartMap_RaisingHooks(self):
    """Tests that MR can be scheduled with a dummy hook class installed.

    The dummy hook class raises NotImplementedError for all method calls so the
    default scheduling logic should be used.

    Most of start_map functionality is already tested by handlers_test.
    Just a smoke test is enough.
    """
    TestEntity().put()

    shard_count = 4
    mapreduce_id = control.start_map(
        "test_map",
        __name__ + ".test_handler",
        "mapreduce.input_readers.DatastoreInputReader",
        {
            "entity_kind": __name__ + "." + TestEntity.__name__,
        },
        shard_count,
        mapreduce_parameters={"foo": "bar"},
        base_path="/mapreduce_base_path",
        queue_name="crazy-queue",
        hooks_class_name=hooks.__name__+"."+hooks.Hooks.__name__)

    self.validate_map_started(mapreduce_id)
开发者ID:akbertram,项目名称:appengine-mapreduce,代码行数:26,代码来源:control_test.py


示例19: testStartMap_Hooks

  def testStartMap_Hooks(self):
    """Tests that MR can be scheduled with a hook class installed.

    Most of start_map functionality is already tested by handlers_test.
    Just a smoke test is enough.
    """
    TestEntity().put()

    shard_count = 4
    mapreduce_id = control.start_map(
        "test_map",
        __name__ + ".test_handler",
        "mapreduce.input_readers.DatastoreInputReader",
        {
            "entity_kind": __name__ + "." + TestEntity.__name__,
        },
        shard_count,
        mapreduce_parameters={"foo": "bar"},
        base_path="/mapreduce_base_path",
        queue_name="crazy-queue",
        hooks_class_name=__name__+"."+TestHooks.__name__)

    self.assertTrue(mapreduce_id)
    task, queue_name = TestHooks.enqueue_kickoff_task_calls[0]
    self.assertEquals("/mapreduce_base_path/kickoffjob_callback", task.url)
    self.assertEquals("crazy-queue", queue_name)
开发者ID:akbertram,项目名称:appengine-mapreduce,代码行数:26,代码来源:control_test.py


示例20: testStartMap_Eta

  def testStartMap_Eta(self):
    """Test that MR can be scheduled into the future.

    Most of start_map functionality is already tested by handlers_test.
    Just a smoke test is enough.
    """
    TestEntity().put()

    # MR should be scheduled into the future.
    eta = datetime.datetime.utcnow() + datetime.timedelta(hours=1)

    shard_count = 4
    mapreduce_id = control.start_map(
        "test_map",
        __name__ + ".test_handler",
        "mapreduce.input_readers.DatastoreInputReader",
        {
            "entity_kind": __name__ + "." + TestEntity.__name__,
        },
        shard_count,
        mapreduce_parameters={"foo": "bar"},
        base_path="/mapreduce_base_path",
        queue_name=self.QUEUE_NAME,
        eta=eta)

    task_eta = self.validate_map_started(mapreduce_id)
    self.assertEquals(eta.strftime("%Y/%m/%d %H:%M:%S"), task_eta)
开发者ID:akbertram,项目名称:appengine-mapreduce,代码行数:27,代码来源:control_test.py



注:本文中的mapreduce.control.start_map函数示例由纯净天空整理自Github/MSDocs等源码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
Python simplejson.loads函数代码示例发布时间:2022-05-27
下一篇:
Python context.get函数代码示例发布时间:2022-05-27
热门推荐
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap