hub.py 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687
  1. # Ultralytics YOLO 🚀, AGPL-3.0 license
  2. import json
  3. from time import time
  4. from ultralytics.hub.utils import HUB_WEB_ROOT, PREFIX, events
  5. from ultralytics.utils import LOGGER, SETTINGS
  6. from ultralytics.utils.torch_utils import model_info_for_loggers
  7. def on_pretrain_routine_end(trainer):
  8. """Logs info before starting timer for upload rate limit."""
  9. session = getattr(trainer, 'hub_session', None)
  10. if session:
  11. # Start timer for upload rate limit
  12. LOGGER.info(f'{PREFIX}View model at {HUB_WEB_ROOT}/models/{session.model_id} 🚀')
  13. session.timers = {'metrics': time(), 'ckpt': time()} # start timer on session.rate_limit
  14. def on_fit_epoch_end(trainer):
  15. """Uploads training progress metrics at the end of each epoch."""
  16. session = getattr(trainer, 'hub_session', None)
  17. if session:
  18. # Upload metrics after val end
  19. all_plots = {**trainer.label_loss_items(trainer.tloss, prefix='train'), **trainer.metrics}
  20. if trainer.epoch == 0:
  21. all_plots = {**all_plots, **model_info_for_loggers(trainer)}
  22. session.metrics_queue[trainer.epoch] = json.dumps(all_plots)
  23. if time() - session.timers['metrics'] > session.rate_limits['metrics']:
  24. session.upload_metrics()
  25. session.timers['metrics'] = time() # reset timer
  26. session.metrics_queue = {} # reset queue
  27. def on_model_save(trainer):
  28. """Saves checkpoints to Ultralytics HUB with rate limiting."""
  29. session = getattr(trainer, 'hub_session', None)
  30. if session:
  31. # Upload checkpoints with rate limiting
  32. is_best = trainer.best_fitness == trainer.fitness
  33. if time() - session.timers['ckpt'] > session.rate_limits['ckpt']:
  34. LOGGER.info(f'{PREFIX}Uploading checkpoint {HUB_WEB_ROOT}/models/{session.model_id}')
  35. session.upload_model(trainer.epoch, trainer.last, is_best)
  36. session.timers['ckpt'] = time() # reset timer
  37. def on_train_end(trainer):
  38. """Upload final model and metrics to Ultralytics HUB at the end of training."""
  39. session = getattr(trainer, 'hub_session', None)
  40. if session:
  41. # Upload final model and metrics with exponential standoff
  42. LOGGER.info(f'{PREFIX}Syncing final model...')
  43. session.upload_model(trainer.epoch, trainer.best, map=trainer.metrics.get('metrics/mAP50-95(B)', 0), final=True)
  44. session.alive = False # stop heartbeats
  45. LOGGER.info(f'{PREFIX}Done ✅\n'
  46. f'{PREFIX}View model at {HUB_WEB_ROOT}/models/{session.model_id} 🚀')
  47. def on_train_start(trainer):
  48. """Run events on train start."""
  49. events(trainer.args)
  50. def on_val_start(validator):
  51. """Runs events on validation start."""
  52. events(validator.args)
  53. def on_predict_start(predictor):
  54. """Run events on predict start."""
  55. events(predictor.args)
  56. def on_export_start(exporter):
  57. """Run events on export start."""
  58. events(exporter.args)
  59. callbacks = {
  60. 'on_pretrain_routine_end': on_pretrain_routine_end,
  61. 'on_fit_epoch_end': on_fit_epoch_end,
  62. 'on_model_save': on_model_save,
  63. 'on_train_end': on_train_end,
  64. 'on_train_start': on_train_start,
  65. 'on_val_start': on_val_start,
  66. 'on_predict_start': on_predict_start,
  67. 'on_export_start': on_export_start} if SETTINGS['hub'] is True else {} # verify enabled