Request: better mechanism of communicating breaking changes when `mp-api` package updates are required

I think this is my first post as alumni :slight_smile: Hi everyone!

I noticed the MPRester just stopped working I think due to a route renaming, but it was non-obvious that an upgrade was required (I think it just gave a 404).

I think perhaps we discussed having a mp-api version required string in the healthcheck route, and checking this on start up so that we could warn the user if their mp-api is below the required version? It’s possible a link to a changelog could be generated if there are updates also, so that people can be informed when an important API change or addition occurs. Just an idea for keeping people in the loop (from the outside I’m realizing how opaque API updates can be).

Best,

Matt

@munrojm and I already put the version check in the MPRester with the appropriate warning before we deployed the API. Unfortunately, that will only cover these scenarios going forward and only for users who have upgraded their client to the currently latest version. We are also in the process of rolling out a notification system that will make it easier for us to communicate with our active users. So we are aware and are doing what we can to improve the process but it was unavoidable to be a little bit of a hiccup this time.

Ok perfect, thanks for the update!

I just ran into this issue too, good to see it’s being taken care of. For reference, here’s the error I was getting before updating my client:

---------------------------------------------------------------------------
MPRestError                               Traceback (most recent call last)
File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:841, in BaseRester.get_data_by_id(self, document_id, fields)
    840 try:
--> 841     results = self._query_resource_data(criteria=criteria, fields=fields, suburl=document_id)  # type: ignore
    842 except MPRestError:

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:799, in BaseRester._query_resource_data(self, criteria, fields, suburl, use_document_model, timeout)
    784 """
    785 Query the endpoint for a list of documents without associated meta information. Only
    786 returns a single page of results.
   (...)
    796     A list of documents
    797 """
--> 799 return self._query_resource(  # type: ignore
    800     criteria=criteria,
    801     fields=fields,
    802     suburl=suburl,
    803     use_document_model=use_document_model,
    804     chunk_size=1000,
    805     num_chunks=1,
    806 ).get("data")

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:297, in BaseRester._query_resource(self, criteria, fields, suburl, use_document_model, parallel_param, num_chunks, chunk_size, timeout)
    295         url += "/"
--> 297 data = self._submit_requests(
    298     url=url,
    299     criteria=criteria,
    300     use_document_model=use_document_model,
    301     parallel_param=parallel_param,
    302     num_chunks=num_chunks,
    303     chunk_size=chunk_size,
    304     timeout=timeout,
    305 )
    307 return data

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:431, in BaseRester._submit_requests(self, url, criteria, use_document_model, parallel_param, num_chunks, chunk_size, timeout)
    429 initial_params_list = [{"url": url, "verify": True, "params": copy(crit)} for crit in new_criteria]
--> 431 initial_data_tuples = self._multi_thread(use_document_model, initial_params_list)
    433 for data, subtotal, crit_ind in initial_data_tuples:

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:636, in BaseRester._multi_thread(self, use_document_model, params_list, progress_bar, timeout)
    634 for future in finished:
--> 636     data, subtotal = future.result()
    638     if progress_bar is not None:

File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:438, in Future.result(self, timeout)
    437 elif self._state == FINISHED:
--> 438     return self.__get_result()
    440 self._condition.wait(timeout)

File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:390, in Future.__get_result(self)
    389 try:
--> 390     raise self._exception
    391 finally:
    392     # Break a reference cycle with the exception in self._exception

File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py:58, in _WorkItem.run(self)
     57 try:
---> 58     result = self.fn(*self.args, **self.kwargs)
     59 except BaseException as exc:

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:724, in BaseRester._submit_request_and_process(self, url, verify, params, use_document_model, timeout)
    722         message = str(data)
--> 724 raise MPRestError(
    725     f"REST query returned with error status code {response.status_code} "
    726     f"on URL {response.url} with message:\n{message}"
    727 )

MPRestError: REST query returned with error status code 404 on URL https://api.materialsproject.org/materials/mp-149/?_limit=1&_fields=structure with message:
Not Found

During handling of the above exception, another exception occurred:

MPRestError                               Traceback (most recent call last)
Cell In[148], line 12
      4 mpr = MPRester(api_key="F3MFJelQpJFdXiGdjtzH9lehJBngwRqz")
      7 # mpr.materials.search(formula="CoSe2")
      8 # 
      9 # for doc in mpr.materials.search(material_ids=["mp-149"]):
     10 #     print(doc)
---> 12 s = mpr.get_structure_by_material_id(material_id="mp-149")
     13 print(s)

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/mprester.py:241, in MPRester.get_structure_by_material_id(self, material_id, final, conventional_unit_cell)
    223 def get_structure_by_material_id(
    224     self, material_id: str, final: bool = True, conventional_unit_cell: bool = False
    225 ) -> Union[Structure, List[Structure]]:
    226     """
    227     Get a Structure corresponding to a material_id.
    228 
   (...)
    238         Structure object or list of Structure objects.
    239     """
--> 241     structure_data = self.materials.get_structure_by_material_id(material_id=material_id, final=final)
    243     if conventional_unit_cell and structure_data:
    244         if final:

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/routes/materials.py:38, in MaterialsRester.get_structure_by_material_id(self, material_id, final)
     25 """
     26 Get a structure for a given Materials Project ID.
     27 
   (...)
     35         pymatgen structure objects.
     36 """
     37 if final:
---> 38     response = self.get_data_by_id(material_id, fields=["structure"])
     39     return response.structure if response is not None else response  # type: ignore
     40 else:

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:857, in BaseRester.get_data_by_id(self, document_id, fields)
    847 from mp_api.client.routes.materials import MaterialsRester
    849 with MaterialsRester(
    850     api_key=self.api_key,
    851     endpoint=self.base_endpoint,
   (...)
    855     headers=self.headers,
    856 ) as mpr:
--> 857     docs = mpr.search(task_ids=[document_id], fields=["material_id"])
    859 if len(docs) > 0:
    861     new_document_id = docs[0].get("material_id", None)

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/routes/materials.py:179, in MaterialsRester.search(self, material_ids, chemsys, crystal_system, density, deprecated, elements, exclude_elements, formula, num_elements, num_sites, spacegroup_number, spacegroup_symbol, task_ids, volume, sort_fields, num_chunks, chunk_size, all_fields, fields)
    169     query_params.update(
    170         {"_sort_fields": ",".join([s.strip() for s in sort_fields])}
    171     )
    173 query_params = {
    174     entry: query_params[entry]
    175     for entry in query_params
    176     if query_params[entry] is not None
    177 }
--> 179 return super()._search(
    180     num_chunks=num_chunks,
    181     chunk_size=chunk_size,
    182     all_fields=all_fields,
    183     fields=fields,
    184     **query_params
    185 )

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:911, in BaseRester._search(self, num_chunks, chunk_size, all_fields, fields, **kwargs)
    888 """
    889 A generic search method to retrieve documents matching specific parameters.
    890 
   (...)
    906     A list of documents.
    907 """
    908 # This method should be customized for each end point to give more user friendly,
    909 # documented kwargs.
--> 911 return self._get_all_documents(
    912     kwargs,
    913     all_fields=all_fields,
    914     fields=fields,
    915     chunk_size=chunk_size,
    916     num_chunks=num_chunks,
    917 )

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:960, in BaseRester._get_all_documents(self, query_params, all_fields, fields, chunk_size, num_chunks)
    946 list_entries = sorted(
    947     (
    948         (key, len(entry.split(",")))
   (...)
    955     reverse=True,
    956 )
    958 chosen_param = list_entries[0][0] if len(list_entries) > 0 else None
--> 960 results = self._query_resource(
    961     query_params,
    962     fields=fields,
    963     parallel_param=chosen_param,
    964     chunk_size=chunk_size,
    965     num_chunks=num_chunks,
    966 )
    968 return results["data"]

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:297, in BaseRester._query_resource(self, criteria, fields, suburl, use_document_model, parallel_param, num_chunks, chunk_size, timeout)
    294         if not url.endswith("/"):
    295             url += "/"
--> 297     data = self._submit_requests(
    298         url=url,
    299         criteria=criteria,
    300         use_document_model=use_document_model,
    301         parallel_param=parallel_param,
    302         num_chunks=num_chunks,
    303         chunk_size=chunk_size,
    304         timeout=timeout,
    305     )
    307     return data
    309 except RequestException as ex:

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:431, in BaseRester._submit_requests(self, url, criteria, use_document_model, parallel_param, num_chunks, chunk_size, timeout)
    427 remaining_docs_avail = {}
    429 initial_params_list = [{"url": url, "verify": True, "params": copy(crit)} for crit in new_criteria]
--> 431 initial_data_tuples = self._multi_thread(use_document_model, initial_params_list)
    433 for data, subtotal, crit_ind in initial_data_tuples:
    435     subtotals.append(subtotal)

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:636, in BaseRester._multi_thread(self, use_document_model, params_list, progress_bar, timeout)
    632 finished, futures = wait(futures, return_when=FIRST_COMPLETED)
    634 for future in finished:
--> 636     data, subtotal = future.result()
    638     if progress_bar is not None:
    639         progress_bar.update(len(data["data"]))

File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:438, in Future.result(self, timeout)
    436     raise CancelledError()
    437 elif self._state == FINISHED:
--> 438     return self.__get_result()
    440 self._condition.wait(timeout)
    442 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:390, in Future.__get_result(self)
    388 if self._exception:
    389     try:
--> 390         raise self._exception
    391     finally:
    392         # Break a reference cycle with the exception in self._exception
    393         self = None

File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/thread.py:58, in _WorkItem.run(self)
     55     return
     57 try:
---> 58     result = self.fn(*self.args, **self.kwargs)
     59 except BaseException as exc:
     60     self.future.set_exception(exc)

File ~/alex/lbl/projects/doping/denv/lib/python3.9/site-packages/mp_api/client/core/client.py:724, in BaseRester._submit_request_and_process(self, url, verify, params, use_document_model, timeout)
    721     except (KeyError, IndexError):
    722         message = str(data)
--> 724 raise MPRestError(
    725     f"REST query returned with error status code {response.status_code} "
    726     f"on URL {response.url} with message:\n{message}"
    727 )

MPRestError: REST query returned with error status code 404 on URL https://api.materialsproject.org/materials/?deprecated=False&_fields=material_id&task_ids=mp-149&_limit=1000 with message:
Not Found

Really confusing and I thought I was doing something wrong, but updating client fixed it. Maybe some sort of warning based on the endpoint would be helpful (if it is not already taken care of for versions going forward)