utils.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223
  1. # Ultralytics YOLO 🚀, AGPL-3.0 license
  2. import os
  3. import platform
  4. import random
  5. import sys
  6. import threading
  7. import time
  8. from pathlib import Path
  9. import requests
  10. from tqdm import tqdm
  11. from ultralytics.utils import (ENVIRONMENT, LOGGER, ONLINE, RANK, SETTINGS, TESTS_RUNNING, TQDM_BAR_FORMAT, TryExcept,
  12. __version__, colorstr, get_git_origin_url, is_colab, is_git_dir, is_pip_package)
  13. from ultralytics.utils.downloads import GITHUB_ASSETS_NAMES
  14. PREFIX = colorstr('Ultralytics HUB: ')
  15. HELP_MSG = 'If this issue persists please visit https://github.com/ultralytics/hub/issues for assistance.'
  16. HUB_API_ROOT = os.environ.get('ULTRALYTICS_HUB_API', 'https://api.ultralytics.com')
  17. HUB_WEB_ROOT = os.environ.get('ULTRALYTICS_HUB_WEB', 'https://hub.ultralytics.com')
  18. def request_with_credentials(url: str) -> any:
  19. """
  20. Make an AJAX request with cookies attached in a Google Colab environment.
  21. Args:
  22. url (str): The URL to make the request to.
  23. Returns:
  24. (any): The response data from the AJAX request.
  25. Raises:
  26. OSError: If the function is not run in a Google Colab environment.
  27. """
  28. if not is_colab():
  29. raise OSError('request_with_credentials() must run in a Colab environment')
  30. from google.colab import output # noqa
  31. from IPython import display # noqa
  32. display.display(
  33. display.Javascript("""
  34. window._hub_tmp = new Promise((resolve, reject) => {
  35. const timeout = setTimeout(() => reject("Failed authenticating existing browser session"), 5000)
  36. fetch("%s", {
  37. method: 'POST',
  38. credentials: 'include'
  39. })
  40. .then((response) => resolve(response.json()))
  41. .then((json) => {
  42. clearTimeout(timeout);
  43. }).catch((err) => {
  44. clearTimeout(timeout);
  45. reject(err);
  46. });
  47. });
  48. """ % url))
  49. return output.eval_js('_hub_tmp')
  50. def requests_with_progress(method, url, **kwargs):
  51. """
  52. Make an HTTP request using the specified method and URL, with an optional progress bar.
  53. Args:
  54. method (str): The HTTP method to use (e.g. 'GET', 'POST').
  55. url (str): The URL to send the request to.
  56. **kwargs (dict): Additional keyword arguments to pass to the underlying `requests.request` function.
  57. Returns:
  58. (requests.Response): The response object from the HTTP request.
  59. Note:
  60. If 'progress' is set to True, the progress bar will display the download progress
  61. for responses with a known content length.
  62. """
  63. progress = kwargs.pop('progress', False)
  64. if not progress:
  65. return requests.request(method, url, **kwargs)
  66. response = requests.request(method, url, stream=True, **kwargs)
  67. total = int(response.headers.get('content-length', 0)) # total size
  68. try:
  69. pbar = tqdm(total=total, unit='B', unit_scale=True, unit_divisor=1024, bar_format=TQDM_BAR_FORMAT)
  70. for data in response.iter_content(chunk_size=1024):
  71. pbar.update(len(data))
  72. pbar.close()
  73. except requests.exceptions.ChunkedEncodingError: # avoid 'Connection broken: IncompleteRead' warnings
  74. response.close()
  75. return response
  76. def smart_request(method, url, retry=3, timeout=30, thread=True, code=-1, verbose=True, progress=False, **kwargs):
  77. """
  78. Makes an HTTP request using the 'requests' library, with exponential backoff retries up to a specified timeout.
  79. Args:
  80. method (str): The HTTP method to use for the request. Choices are 'post' and 'get'.
  81. url (str): The URL to make the request to.
  82. retry (int, optional): Number of retries to attempt before giving up. Default is 3.
  83. timeout (int, optional): Timeout in seconds after which the function will give up retrying. Default is 30.
  84. thread (bool, optional): Whether to execute the request in a separate daemon thread. Default is True.
  85. code (int, optional): An identifier for the request, used for logging purposes. Default is -1.
  86. verbose (bool, optional): A flag to determine whether to print out to console or not. Default is True.
  87. progress (bool, optional): Whether to show a progress bar during the request. Default is False.
  88. **kwargs (dict): Keyword arguments to be passed to the requests function specified in method.
  89. Returns:
  90. (requests.Response): The HTTP response object. If the request is executed in a separate thread, returns None.
  91. """
  92. retry_codes = (408, 500) # retry only these codes
  93. @TryExcept(verbose=verbose)
  94. def func(func_method, func_url, **func_kwargs):
  95. """Make HTTP requests with retries and timeouts, with optional progress tracking."""
  96. r = None # response
  97. t0 = time.time() # initial time for timer
  98. for i in range(retry + 1):
  99. if (time.time() - t0) > timeout:
  100. break
  101. r = requests_with_progress(func_method, func_url, **func_kwargs) # i.e. get(url, data, json, files)
  102. if r.status_code < 300: # return codes in the 2xx range are generally considered "good" or "successful"
  103. break
  104. try:
  105. m = r.json().get('message', 'No JSON message.')
  106. except AttributeError:
  107. m = 'Unable to read JSON.'
  108. if i == 0:
  109. if r.status_code in retry_codes:
  110. m += f' Retrying {retry}x for {timeout}s.' if retry else ''
  111. elif r.status_code == 429: # rate limit
  112. h = r.headers # response headers
  113. m = f"Rate limit reached ({h['X-RateLimit-Remaining']}/{h['X-RateLimit-Limit']}). " \
  114. f"Please retry after {h['Retry-After']}s."
  115. if verbose:
  116. LOGGER.warning(f'{PREFIX}{m} {HELP_MSG} ({r.status_code} #{code})')
  117. if r.status_code not in retry_codes:
  118. return r
  119. time.sleep(2 ** i) # exponential standoff
  120. return r
  121. args = method, url
  122. kwargs['progress'] = progress
  123. if thread:
  124. threading.Thread(target=func, args=args, kwargs=kwargs, daemon=True).start()
  125. else:
  126. return func(*args, **kwargs)
  127. class Events:
  128. """
  129. A class for collecting anonymous event analytics. Event analytics are enabled when sync=True in settings and
  130. disabled when sync=False. Run 'yolo settings' to see and update settings YAML file.
  131. Attributes:
  132. url (str): The URL to send anonymous events.
  133. rate_limit (float): The rate limit in seconds for sending events.
  134. metadata (dict): A dictionary containing metadata about the environment.
  135. enabled (bool): A flag to enable or disable Events based on certain conditions.
  136. """
  137. url = 'https://www.google-analytics.com/mp/collect?measurement_id=G-X8NCJYTQXM&api_secret=QLQrATrNSwGRFRLE-cbHJw'
  138. def __init__(self):
  139. """
  140. Initializes the Events object with default values for events, rate_limit, and metadata.
  141. """
  142. self.events = [] # events list
  143. self.rate_limit = 60.0 # rate limit (seconds)
  144. self.t = 0.0 # rate limit timer (seconds)
  145. self.metadata = {
  146. 'cli': Path(sys.argv[0]).name == 'yolo',
  147. 'install': 'git' if is_git_dir() else 'pip' if is_pip_package() else 'other',
  148. 'python': '.'.join(platform.python_version_tuple()[:2]), # i.e. 3.10
  149. 'version': __version__,
  150. 'env': ENVIRONMENT,
  151. 'session_id': round(random.random() * 1E15),
  152. 'engagement_time_msec': 1000}
  153. self.enabled = \
  154. SETTINGS['sync'] and \
  155. RANK in (-1, 0) and \
  156. not TESTS_RUNNING and \
  157. ONLINE and \
  158. (is_pip_package() or get_git_origin_url() == 'https://github.com/ultralytics/ultralytics.git')
  159. def __call__(self, cfg):
  160. """
  161. Attempts to add a new event to the events list and send events if the rate limit is reached.
  162. Args:
  163. cfg (IterableSimpleNamespace): The configuration object containing mode and task information.
  164. """
  165. if not self.enabled:
  166. # Events disabled, do nothing
  167. return
  168. # Attempt to add to events
  169. if len(self.events) < 25: # Events list limited to 25 events (drop any events past this)
  170. params = {
  171. **self.metadata, 'task': cfg.task,
  172. 'model': cfg.model if cfg.model in GITHUB_ASSETS_NAMES else 'custom'}
  173. if cfg.mode == 'export':
  174. params['format'] = cfg.format
  175. self.events.append({'name': cfg.mode, 'params': params})
  176. # Check rate limit
  177. t = time.time()
  178. if (t - self.t) < self.rate_limit:
  179. # Time is under rate limiter, wait to send
  180. return
  181. # Time is over rate limiter, send now
  182. data = {'client_id': SETTINGS['uuid'], 'events': self.events} # SHA-256 anonymized UUID hash and events list
  183. # POST equivalent to requests.post(self.url, json=data)
  184. smart_request('post', self.url, json=data, retry=0, verbose=False)
  185. # Reset events and rate limit timer
  186. self.events = []
  187. self.t = t
  188. # Run below code on hub/utils init -------------------------------------------------------------------------------------
  189. events = Events()